From c87ecb907deb8481a88815b4af181c3676f7ed05 Mon Sep 17 00:00:00 2001 From: Pete Gadomski Date: Thu, 13 Feb 2025 09:19:22 -0700 Subject: [PATCH] feat(v2): first v1 test (#1523) * feat: first v1 test * ci: run CI on v2 pull requests * feat: add assets mixin * fix: only try to deploy if we're on v2 branch --- .github/workflows/ci.yaml | 4 + pyproject.toml | 3 +- src/pystac/asset.py | 9 + src/pystac/extent.py | 26 ++- src/pystac/item.py | 4 +- src/pystac/stac_object.py | 23 +- tests/test_extent.py | 13 +- tests/test_item.py | 7 + tests/v1/__init__.py | 0 tests/v1/conftest.py | 14 ++ tests/v1/data-files/item/sample-item.json | 81 +++++++ tests/v1/test_item.py | 28 +++ tests/v1/utils.py | 252 ++++++++++++++++++++++ uv.lock | 13 ++ 14 files changed, 466 insertions(+), 11 deletions(-) create mode 100644 tests/v1/__init__.py create mode 100644 tests/v1/conftest.py create mode 100644 tests/v1/data-files/item/sample-item.json create mode 100644 tests/v1/test_item.py create mode 100644 tests/v1/utils.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e2cc351de..b4fd18636 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,6 +4,9 @@ on: push: branches: - v2 + pull_request: + branches: + - v2 jobs: test: @@ -52,6 +55,7 @@ jobs: path: site/ deploy-docs: name: Deploy docs + if: github.ref == 'refs/heads/v2' environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} diff --git a/pyproject.toml b/pyproject.toml index cb5811362..14e792d72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ "Programming Language :: Python :: 3.13", ] requires-python = ">=3.10" -dependencies = ["typing-extensions>=4.12.2"] +dependencies = ["python-dateutil>=2.9.0.post0", "typing-extensions>=4.12.2"] [project.optional-dependencies] validate = ["jsonschema>=4.23.0", "referencing>=0.36.2"] @@ -35,6 +35,7 @@ dev = [ "pytest>=8.3.4", "ruff>=0.9.6", "types-jsonschema>=4.23.0.20241208", + "types-python-dateutil>=2.9.0.20241206", ] bench = ["asv>=0.6.4"] docs = ["mike>=2.1.3", "mkdocs-material>=9.6.3", "mkdocstrings-python>=1.14.6"] diff --git a/src/pystac/asset.py b/src/pystac/asset.py index 1464cfeed..62f284b14 100644 --- a/src/pystac/asset.py +++ b/src/pystac/asset.py @@ -75,3 +75,12 @@ def to_dict(self) -> dict[str, Any]: d = {"href": self.href} d.update(super().to_dict()) return d + + +class AssetsMixin: + """A mixin for things that have assets (Collections and Items)""" + + assets: dict[str, Asset] + + def add_asset(self, key: str, asset: Asset) -> None: + raise NotImplementedError diff --git a/src/pystac/extent.py b/src/pystac/extent.py index fa714dc66..ee5c7a6cf 100644 --- a/src/pystac/extent.py +++ b/src/pystac/extent.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy import datetime import warnings from typing import Any, Sequence @@ -7,6 +8,7 @@ from typing_extensions import Self from .constants import DEFAULT_BBOX, DEFAULT_INTERVAL +from .decorators import v2_deprecated from .errors import StacWarning from .types import PermissiveBbox, PermissiveInterval @@ -54,7 +56,19 @@ def from_dict(cls: type[Self], d: dict[str, Any]) -> Self: """Creates a new spatial extent from a dictionary.""" return cls(**d) - def __init__(self, bbox: PermissiveBbox | None = None): + @classmethod + @v2_deprecated("Use the constructor instead") + def from_coordinates( + cls: type[Self], + coordinates: list[Any], + extra_fields: dict[str, Any] | None = None, + ) -> Self: + if extra_fields: + return cls(coordinates, **extra_fields) + else: + return cls(coordinates) + + def __init__(self, bbox: PermissiveBbox | None = None, **kwargs: Any): """Creates a new spatial extent.""" self.bbox: Sequence[Sequence[float | int]] if bbox is None or len(bbox) == 0: @@ -63,10 +77,13 @@ def __init__(self, bbox: PermissiveBbox | None = None): self.bbox = bbox # type: ignore else: self.bbox = [bbox] # type: ignore + self.extra_fields = kwargs def to_dict(self) -> dict[str, Any]: """Converts this spatial extent to a dictionary.""" - return {"bbox": self.bbox} + d = copy.deepcopy(self.extra_fields) + d["bbox"] = self.bbox + return d class TemporalExtent: @@ -77,6 +94,11 @@ def from_dict(cls: type[Self], d: dict[str, Any]) -> Self: """Creates a new temporal extent from a dictionary.""" return cls(**d) + @classmethod + def from_now(cls: type[Self]) -> Self: + """Creates a new temporal extent that starts now and has no end time.""" + return cls([[datetime.datetime.now(tz=datetime.timezone.utc), None]]) + def __init__( self, interval: PermissiveInterval | None = None, diff --git a/src/pystac/item.py b/src/pystac/item.py index 9874c36ac..618e66ce5 100644 --- a/src/pystac/item.py +++ b/src/pystac/item.py @@ -5,14 +5,14 @@ import warnings from typing import Any, Sequence -from .asset import Asset +from .asset import Asset, AssetsMixin from .constants import ITEM_TYPE from .errors import StacWarning from .link import Link from .stac_object import STACObject -class Item(STACObject): +class Item(STACObject, AssetsMixin): """An Item is a GeoJSON Feature augmented with foreign members relevant to a STAC object. diff --git a/src/pystac/stac_object.py b/src/pystac/stac_object.py index cf5f12002..412f7557a 100644 --- a/src/pystac/stac_object.py +++ b/src/pystac/stac_object.py @@ -21,6 +21,7 @@ from .link import Link if TYPE_CHECKING: + from .catalog import Catalog from .io import Read, Write @@ -77,13 +78,16 @@ def from_file( @classmethod def from_dict( - cls: type[STACObject], + cls: type[Self], d: dict[str, Any], *, href: str | None = None, + root: Catalog | None = None, # TODO deprecation warning + migrate: bool = False, + preserve_dict: bool = True, # TODO deprecation warning reader: Read | None = None, writer: Write | None = None, - ) -> STACObject: + ) -> Self: """Creates a STAC object from a dictionary. If you already know what type of STAC object your dictionary represents, @@ -108,17 +112,24 @@ def from_dict( if type_value == CATALOG_TYPE: from .catalog import Catalog - return Catalog(**d, href=href, reader=reader, writer=writer) + stac_object: STACObject = Catalog( + **d, href=href, reader=reader, writer=writer + ) elif type_value == COLLECTION_TYPE: from .collection import Collection - return Collection(**d, href=href, reader=reader, writer=writer) + stac_object = Collection(**d, href=href, reader=reader, writer=writer) elif type_value == ITEM_TYPE: from .item import Item - return Item(**d, href=href, reader=reader, writer=writer) + stac_object = Item(**d, href=href, reader=reader, writer=writer) else: raise StacError(f"unknown type field: {type_value}") + + if isinstance(stac_object, cls): + return stac_object + else: + raise PystacError(f"Expected {cls} but got a {type(stac_object)}") else: raise StacError("missing type field on dictionary") @@ -136,6 +147,8 @@ def __init__( """Creates a new STAC object.""" from .extensions import Extensions + super().__init__() + self.id: str = id """The object's id.""" diff --git a/tests/test_extent.py b/tests/test_extent.py index fa0121828..f6323aa9d 100644 --- a/tests/test_extent.py +++ b/tests/test_extent.py @@ -3,7 +3,7 @@ import pytest -from pystac import StacWarning, TemporalExtent +from pystac import SpatialExtent, StacWarning, TemporalExtent def test_temporal_with_datetimes() -> None: @@ -45,3 +45,14 @@ def test_temporal_with_bad_tail() -> None: ) d = extent.to_dict() assert d == {"interval": [["2025-02-11T00:00:00Z", None]]} + + +def test_temporal_from_now() -> None: + extent = TemporalExtent.from_now() + assert isinstance(extent.interval[0][0], str) + assert extent.interval[0][1] is None + + +def test_spatial_from_coordinates() -> None: + with pytest.warns(FutureWarning): + SpatialExtent.from_coordinates([-180, -90, 180, 90]) diff --git a/tests/test_item.py b/tests/test_item.py index d16a6f9dc..1449d440d 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -37,3 +37,10 @@ def test_warn_include_self_link() -> None: def test_warn_transform_hrefs() -> None: with pytest.warns(FutureWarning): Item("an-id").to_dict(transform_hrefs=True) + + +def test_from_dict_migrate() -> None: + d = Item("an-id").to_dict() + d["stac_version"] = "1.0.0" + item = Item.from_dict(d, migrate=True) + item.stac_version == "1.1.0" diff --git a/tests/v1/__init__.py b/tests/v1/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/v1/conftest.py b/tests/v1/conftest.py new file mode 100644 index 000000000..0722161f9 --- /dev/null +++ b/tests/v1/conftest.py @@ -0,0 +1,14 @@ +import json +from typing import Any + +import pytest + +from .utils import TestCases + + +@pytest.fixture +def sample_item_dict() -> dict[str, Any]: + m = TestCases.get_path("data-files/item/sample-item.json") + with open(m) as f: + item_dict: dict[str, Any] = json.load(f) + return item_dict diff --git a/tests/v1/data-files/item/sample-item.json b/tests/v1/data-files/item/sample-item.json new file mode 100644 index 000000000..7319b8275 --- /dev/null +++ b/tests/v1/data-files/item/sample-item.json @@ -0,0 +1,81 @@ +{ + "type": "Feature", + "stac_version": "1.1.0", + "id": "CS3-20160503_132131_05", + "properties": { + "datetime": "2016-05-03T13:22:30.040000Z", + "title": "A CS3 item", + "license": "PDDL-1.0", + "providers": [ + { + "name": "CoolSat", + "roles": [ + "producer", + "licensor" + ], + "url": "https://cool-sat.com/" + } + ] + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -122.308150179, + 37.488035566 + ], + [ + -122.597502109, + 37.538869539 + ], + [ + -122.576687533, + 37.613537207 + ], + [ + -122.2880486, + 37.562818007 + ], + [ + -122.308150179, + 37.488035566 + ] + ] + ] + }, + "links": [ + { + "rel": "collection", + "href": "https://raw.githubusercontent.com/radiantearth/stac-spec/v0.8.1/collection-spec/examples/sentinel2.json" + } + ], + "assets": { + "analytic": { + "href": "http://cool-sat.com/catalog/CS3-20160503_132130_04/analytic.tif", + "title": "4-Band Analytic", + "product": "http://cool-sat.com/catalog/products/analytic.json", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "analytic" + ] + }, + "thumbnail": { + "href": "http://cool-sat.com/catalog/CS3-20160503_132130_04/thumbnail.png", + "title": "Thumbnail", + "type": "image/png", + "roles": [ + "thumbnail" + ] + } + }, + "bbox": [ + -122.59750209, + 37.48803556, + -122.2880486, + 37.613537207 + ], + "stac_extensions": [], + "collection": "CS3" +} \ No newline at end of file diff --git a/tests/v1/test_item.py b/tests/v1/test_item.py new file mode 100644 index 000000000..ef9ca0927 --- /dev/null +++ b/tests/v1/test_item.py @@ -0,0 +1,28 @@ +from copy import deepcopy +from typing import Any + +from pystac import Item + +from . import utils + + +def test_to_from_dict(sample_item_dict: dict[str, Any]) -> None: + param_dict = deepcopy(sample_item_dict) + + utils.assert_to_from_dict(Item, param_dict) + item = Item.from_dict(param_dict) + assert item.id == "CS3-20160503_132131_05" + + # test asset creation additional field(s) + assert ( + item.assets["analytic"].extra_fields["product"] + == "http://cool-sat.com/catalog/products/analytic.json" + ) + assert len(item.assets["thumbnail"].extra_fields) == 0 + + # test that the parameter is preserved + assert param_dict == sample_item_dict + + # assert that the parameter is preserved regardless of preserve_dict + Item.from_dict(param_dict, preserve_dict=False) + assert param_dict == sample_item_dict diff --git a/tests/v1/utils.py b/tests/v1/utils.py new file mode 100644 index 000000000..f06b3564b --- /dev/null +++ b/tests/v1/utils.py @@ -0,0 +1,252 @@ +import csv +import datetime +import os +from copy import deepcopy +from pathlib import Path +from typing import Any + +import pytest +from dateutil.parser import parse + +import pystac +from pystac import ( + Asset, + Catalog, + Collection, + Extent, + Item, + MediaType, + SpatialExtent, + STACObject, + TemporalExtent, +) + +ARBITRARY_GEOM: dict[str, Any] = { + "type": "Polygon", + "coordinates": [ + [ + [-2.5048828125, 3.8916575492899987], + [-1.9610595703125, 3.8916575492899987], + [-1.9610595703125, 4.275202171119132], + [-2.5048828125, 4.275202171119132], + [-2.5048828125, 3.8916575492899987], + ] + ], +} + +ARBITRARY_BBOX: list[float] = [ + ARBITRARY_GEOM["coordinates"][0][0][0], + ARBITRARY_GEOM["coordinates"][0][0][1], + ARBITRARY_GEOM["coordinates"][0][1][0], + ARBITRARY_GEOM["coordinates"][0][1][1], +] + +with pytest.warns(FutureWarning): + ARBITRARY_EXTENT = Extent( + spatial=SpatialExtent.from_coordinates(ARBITRARY_GEOM["coordinates"]), + temporal=TemporalExtent.from_now(), + ) + + +def assert_to_from_dict( + stac_object_class: type[STACObject], + d: dict[str, Any], +) -> None: + def _parse_times(a_dict: dict[str, Any]) -> None: + for k, v in a_dict.items(): + if isinstance(v, dict): + _parse_times(v) + elif isinstance(v, (tuple, list, set)): + for vv in v: + if isinstance(vv, dict): + _parse_times(vv) + else: + if k == "datetime": + if not isinstance(v, datetime.datetime): + a_dict[k] = parse(v) + a_dict[k] = a_dict[k].replace(microsecond=0) + + d1 = deepcopy(d) + d2 = stac_object_class.from_dict(d, migrate=False).to_dict() + _parse_times(d1) + _parse_times(d2) + assert d1 == d2 + + +TEST_LABEL_CATALOG = { + "country-1": { + "area-1-1": { + "dsm": "area-1-1_dsm.tif", + "ortho": "area-1-1_ortho.tif", + "labels": "area-1-1_labels.geojson", + }, + "area-1-2": { + "dsm": "area-1-2_dsm.tif", + "ortho": "area-1-2_ortho.tif", + "labels": "area-1-2_labels.geojson", + }, + }, + "country-2": { + "area-2-1": { + "dsm": "area-2-1_dsm.tif", + "ortho": "area-2-1_ortho.tif", + "labels": "area-2-1_labels.geojson", + }, + "area-2-2": { + "dsm": "area-2-2_dsm.tif", + "ortho": "area-2-2_ortho.tif", + "labels": "area-2-2_labels.geojson", + }, + }, +} + + +class ExampleInfo: + def __init__( + self, + path: str, + object_type: pystac.STACObjectType, + stac_version: str, + extensions: list[str], + valid: bool, + ) -> None: + self.path = path + self.object_type = object_type + self.stac_version = stac_version + self.extensions = extensions + self.valid = valid + + +class TestCases: + bad_catalog_case = "data-files/catalogs/invalid-catalog/catalog.json" + + @staticmethod + def get_path(rel_path: str) -> str: + return str(Path(__file__).parent.joinpath(rel_path)) + + @staticmethod + def get_examples_info() -> list[ExampleInfo]: + examples: list[ExampleInfo] = [] + + info_path = TestCases.get_path("data-files/examples/example-info.csv") + with open(TestCases.get_path("data-files/examples/example-info.csv")) as f: + for row in csv.reader(f): + path = os.path.abspath(os.path.join(os.path.dirname(info_path), row[0])) + object_type = row[1] + stac_version = row[2] + extensions: list[str] = [] + if row[3]: + extensions = row[3].split("|") + + valid = True + if len(row) > 4: + # The 5th column will be "INVALID" if the example + # shouldn't pass validation + valid = row[4] != "INVALID" + + examples.append( + ExampleInfo( + path=path, + object_type=pystac.STACObjectType(object_type), + stac_version=stac_version, + extensions=extensions, + valid=valid, + ) + ) + return examples + + @staticmethod + def all_test_catalogs() -> list[Catalog]: + return [ + TestCases.case_1(), + TestCases.case_2(), + TestCases.case_3(), + TestCases.case_4(), + TestCases.case_5(), + TestCases.case_7(), + TestCases.case_8(), # type: ignore + ] + + @staticmethod + def case_1() -> Catalog: + return Catalog.from_file( + TestCases.get_path("data-files/catalogs/test-case-1/catalog.json") + ) + + @staticmethod + def case_2() -> Catalog: + return Catalog.from_file( + TestCases.get_path("data-files/catalogs/test-case-2/catalog.json") + ) + + @staticmethod + def case_3() -> Catalog: + root_cat = Catalog( + id="test3", description="test case 3 catalog", title="test case 3 title" + ) + + image_item = Item( + id="imagery-item", + geometry=ARBITRARY_GEOM, + bbox=ARBITRARY_BBOX, + datetime=datetime.datetime.now(datetime.timezone.utc), + properties={}, + ) + + image_item.add_asset( + "ortho", Asset(href="some/geotiff.tiff", media_type=MediaType.GEOTIFF) + ) + + label_item = Item( + id="label-items", + geometry=ARBITRARY_GEOM, + bbox=ARBITRARY_BBOX, + datetime=datetime.datetime.now(datetime.timezone.utc), + properties={}, + ) + + root_cat.add_item(image_item) + root_cat.add_item(label_item) + + return root_cat + + @staticmethod + def case_4() -> Catalog: + """Test case that is based on a local copy of the Tier 1 dataset from + DrivenData's OpenCities AI Challenge. + See: https://www.drivendata.org/competitions/60/building-segmentation-disaster\ +-resilience + """ + return Catalog.from_file( + TestCases.get_path("data-files/catalogs/test-case-4/catalog.json") + ) + + @staticmethod + def case_5() -> Catalog: + """Based on a subset of https://cbers.stac.cloud/""" + return Catalog.from_file( + TestCases.get_path("data-files/catalogs/test-case-5/catalog.json") + ) + + @staticmethod + def case_6() -> Catalog: + """Based on a subset of CBERS, contains a root and 4 empty children""" + return Catalog.from_file( + TestCases.get_path("data-files/catalogs/cbers-partial/catalog.json") + ) + + @staticmethod + def case_7() -> Catalog: + """Test case 4 as STAC version 0.8.1""" + return Catalog.from_file( + TestCases.get_path("data-files/catalogs/label_catalog-v0.8.1/catalog.json") + ) + + @staticmethod + def case_8() -> Collection: + """Planet disaster data example catalog, 1.0.0-beta.2""" + return Collection.from_file( + TestCases.get_path( + "data-files/catalogs/planet-example-v1.0.0-beta.2/collection.json" + ) + ) diff --git a/uv.lock b/uv.lock index f16739f35..4397d3983 100644 --- a/uv.lock +++ b/uv.lock @@ -763,6 +763,7 @@ name = "pystac" version = "2.0.0a0" source = { editable = "." } dependencies = [ + { name = "python-dateutil" }, { name = "typing-extensions" }, ] @@ -784,6 +785,7 @@ dev = [ { name = "pytest" }, { name = "ruff" }, { name = "types-jsonschema" }, + { name = "types-python-dateutil" }, ] docs = [ { name = "mike" }, @@ -795,6 +797,7 @@ docs = [ requires-dist = [ { name = "jsonschema", marker = "extra == 'validate'", specifier = ">=4.23.0" }, { name = "obstore", marker = "extra == 'obstore'", specifier = ">=0.4.0" }, + { name = "python-dateutil", specifier = ">=2.9.0.post0" }, { name = "referencing", marker = "extra == 'validate'", specifier = ">=0.36.2" }, { name = "typing-extensions", specifier = ">=4.12.2" }, ] @@ -806,6 +809,7 @@ dev = [ { name = "pytest", specifier = ">=8.3.4" }, { name = "ruff", specifier = ">=0.9.6" }, { name = "types-jsonschema", specifier = ">=4.23.0.20241208" }, + { name = "types-python-dateutil", specifier = ">=2.9.0.20241206" }, ] docs = [ { name = "mike", specifier = ">=2.1.3" }, @@ -1151,6 +1155,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/91/64/4b2fba8b7cb0104ba013f2a1bf6f39a98e927e14befe1ef947d373b25218/types_jsonschema-4.23.0.20241208-py3-none-any.whl", hash = "sha256:87934bd9231c99d8eff94cacfc06ba668f7973577a9bd9e1f9de957c5737313e", size = 15021 }, ] +[[package]] +name = "types-python-dateutil" +version = "2.9.0.20241206" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/60/47d92293d9bc521cd2301e423a358abfac0ad409b3a1606d8fbae1321961/types_python_dateutil-2.9.0.20241206.tar.gz", hash = "sha256:18f493414c26ffba692a72369fea7a154c502646301ebfe3d56a04b3767284cb", size = 13802 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/b3/ca41df24db5eb99b00d97f89d7674a90cb6b3134c52fb8121b6d8d30f15c/types_python_dateutil-2.9.0.20241206-py3-none-any.whl", hash = "sha256:e248a4bc70a486d3e3ec84d0dc30eec3a5f979d6e7ee4123ae043eedbb987f53", size = 14384 }, +] + [[package]] name = "typing-extensions" version = "4.12.2"