From 99812085b2f996ecc614b26b11704c52687ca882 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Tue, 11 Aug 2026 16:38:31 +0200 Subject: [PATCH 01/14] check changes --- deep_code/tests/tools/test_publish.py | 6 +- .../utils/test_dataset_stac_generator.py | 47 +- deep_code/tools/new.py | 11 +- deep_code/tools/prr.py | 30 +- deep_code/tools/publish.py | 42 +- deep_code/utils/dataset_stac_generator.py | 545 ++++++++++++------ docs/cli.md | 5 +- docs/configuration.md | 42 +- docs/python-api.md | 8 +- 9 files changed, 510 insertions(+), 226 deletions(-) diff --git a/deep_code/tests/tools/test_publish.py b/deep_code/tests/tools/test_publish.py index 6ccc8eb..4df212e 100644 --- a/deep_code/tests/tools/test_publish.py +++ b/deep_code/tests/tools/test_publish.py @@ -295,7 +295,9 @@ def test_publish_writes_zarr_stac_to_s3_when_configured( mock_generator = MagicMock() mock_generator.build_zarr_stac_catalog_file_dict.return_value = { "s3://test-bucket/stac/catalog.json": {"type": "Catalog"}, - "s3://test-bucket/stac/test-collection/item.json": {"type": "Feature"}, + "s3://test-bucket/stac/test-collection/items/test-collection.json": { + "type": "Feature" + }, } # Simulate what publish_dataset() normally does: store the generator self.publisher._last_generator = mock_generator @@ -401,7 +403,7 @@ def test_publish_dataset_raises_when_no_dataset_config(self): def test_publish_dataset_raises_when_ids_missing(self): self.publisher.dataset_config = {"collection_id": "", "dataset_id": ""} - with pytest.raises(ValueError, match="Dataset ID or Collection ID missing"): + with pytest.raises(ValueError, match="item configuration"): self.publisher.publish_dataset(write_to_file=False) def test_publish_dataset_raises_when_license_missing(self): diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index e981df6..5a20808 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -24,7 +24,11 @@ VARIABLE_BASE_CATALOG_SELF_HREF, ZARR_MEDIA_TYPE, ) -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator, Theme +from deep_code.utils.dataset_stac_generator import ( + ItemConfig, + OscDatasetStacGenerator, + Theme, +) class TestOSCProductSTACGenerator(unittest.TestCase): @@ -456,7 +460,7 @@ def test_build_zarr_stac_item_structure(self): # Self href self.assertEqual( item.self_href, - "s3://test-bucket/stac/my-collection/mock-collection-id/item.json", + "s3://test-bucket/stac/my-collection/items/mock-collection-id.json", ) # Required link rels @@ -490,7 +494,7 @@ def test_build_zarr_stac_catalog_file_dict_keys(self): catalog_path = "s3://test-bucket/stac/my-collection/catalog.json" item_path = ( - "s3://test-bucket/stac/my-collection/mock-collection-id/item.json" + "s3://test-bucket/stac/my-collection/items/mock-collection-id.json" ) self.assertIn(catalog_path, file_dict) self.assertIn(item_path, file_dict) @@ -506,7 +510,7 @@ def test_build_zarr_stac_catalog_file_dict_content(self): self.assertEqual(catalog_dict["id"], "mock-collection-id-stac-catalog") item_dict = file_dict[ - "s3://test-bucket/stac/my-collection/mock-collection-id/item.json" + "s3://test-bucket/stac/my-collection/items/mock-collection-id.json" ] self.assertEqual(item_dict["type"], "Feature") self.assertEqual(item_dict["id"], "mock-collection-id") @@ -514,6 +518,37 @@ def test_build_zarr_stac_catalog_file_dict_content(self): self.assertIn("zarr-data", item_dict["assets"]) self.assertIn("zarr-consolidated-metadata", item_dict["assets"]) + @patch("deep_code.utils.dataset_stac_generator.open_dataset") + def test_build_zarr_stac_catalog_file_dict_multiple_items(self, mock_open_ds): + """Multiple item configurations produce multiple item files and links.""" + mock_open_ds.return_value = self.mock_dataset + gen = OscDatasetStacGenerator( + collection_id="multi-collection", + items_config=[ + ItemConfig(dataset_id="first.zarr", item_id="first-item"), + ItemConfig(dataset_id="second.zarr", item_id="second-item"), + ], + workflow_id="dummy", + workflow_title="test", + license_type="proprietary", + ) + + file_dict = gen.build_zarr_stac_catalog_file_dict( + "s3://test-bucket/stac/multi-collection/" + ) + + self.assertIn( + "s3://test-bucket/stac/multi-collection/items/first-item.json", + file_dict, + ) + self.assertIn( + "s3://test-bucket/stac/multi-collection/items/second-item.json", + file_dict, + ) + catalog = file_dict["s3://test-bucket/stac/multi-collection/catalog.json"] + item_links = [lnk for lnk in catalog["links"] if lnk["rel"] == "item"] + self.assertEqual(len(item_links), 2) + def test_build_dataset_stac_collection_adds_s3_catalog_via_link(self): """A 'via' link (STAC browser) and a 'child' link (HTTPS catalog) are added when stac_catalog_s3_root is provided. @@ -1019,7 +1054,9 @@ def test_save_prr_collection_writes_tree(self): self.assertEqual(out, tmp) collection_path = os.path.join(tmp, "collection.json") - item_path = os.path.join(tmp, "prr-collection", "prr-collection.json") + item_path = os.path.join( + tmp, "prr-collection", "items", "prr-collection.json" + ) self.assertTrue(os.path.isfile(collection_path)) self.assertTrue(os.path.isfile(item_path)) diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index d910db0..ee1d651 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -56,10 +56,15 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: """Generate a complete dataset template with all possible keys and placeholder values""" required = { - "dataset_id": "[REQUIRED: name of the Zarr store in your S3 bucket, e.g. my-dataset.zarr]", "collection_id": "[REQUIRED: unique identifier, no spaces — use hyphens (e.g. My-Dataset-2024)]", "license_type": "[REQUIRED: SPDX license identifier, e.g. CC-BY-4.0, MIT, proprietary]", "stac_catalog_s3_root": "[REQUIRED: S3 root for the STAC Catalog + Item, e.g. s3://my-bucket/stac/my-collection/]", + "items_config": [ + { + "dataset_id": "[REQUIRED: name of the Zarr store in your S3 bucket, e.g. my-dataset.zarr]", + "item_id": "[REQUIRED: unique STAC item id, no spaces — use hyphens]", + } + ], } optional = { @@ -95,7 +100,9 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: stac_catalog_comment = ( "\n# stac_catalog_s3_root: deep-code writes the following files to this S3 root:\n" "# {stac_catalog_s3_root}/catalog.json (STAC Catalog root)\n" - "# {stac_catalog_s3_root}/{collection_id}/item.json (STAC Item for the whole Zarr)\n" + "# {stac_catalog_s3_root}/{collection_id}/items/{item_id}.json (STAC Item for each Zarr)\n" + "# items_config can contain multiple dataset/item pairs, but publish\n" + "# currently only consumes one item configuration.\n" "# S3 write credentials are resolved in order:\n" "# 1. STAC_S3_KEY / STAC_S3_SECRET env vars (STAC-specific, any bucket)\n" "# 2. AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars\n" diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index 9bfddac..5eff849 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -16,7 +16,7 @@ import fsspec import yaml -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ItemConfig, OscDatasetStacGenerator logger = logging.getLogger(__name__) @@ -38,14 +38,30 @@ def generate_prr_collection( with fsspec.open(dataset_config_path, "r") as file: config = yaml.safe_load(file) or {} - dataset_id = config.get("dataset_id") collection_id = config.get("collection_id") license_type = config.get("license_type") + items_config_raw = config.get("items_config") - if not dataset_id or not collection_id: - raise ValueError( - "Both 'dataset_id' and 'collection_id' are required in the dataset config." - ) + if items_config_raw: + items_config = [ + ItemConfig( + dataset_id=item_config["dataset_id"], + item_id=item_config["item_id"], + ) + for item_config in items_config_raw + ] + else: + dataset_id = config.get("dataset_id") + item_id = config.get("item_id") or collection_id + if not dataset_id or not collection_id: + raise ValueError( + "At least one item configuration must be provided in the dataset config, " + "along with 'collection_id'." + ) + items_config = [ItemConfig(dataset_id=dataset_id, item_id=item_id)] + + if not collection_id: + raise ValueError("collection_id is required in the dataset config.") if not license_type: raise ValueError( "license_type is required in the dataset config. " @@ -54,7 +70,7 @@ def generate_prr_collection( logger.info(f"Generating PRR STAC collection for '{collection_id}'.") generator = OscDatasetStacGenerator( - dataset_id=dataset_id, + items_config=items_config, collection_id=collection_id, workflow_id=config.get("workflow_id") or "", workflow_title=config.get("workflow_title") or "", diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index 3b62cbd..79a68dd 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -22,7 +22,7 @@ OSC_REPO_OWNER, WORKFLOW_BASE_CATALOG_SELF_HREF, ) -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ItemConfig, OscDatasetStacGenerator from deep_code.utils.github_automation import GitHubAutomation from deep_code.utils.ogc_api_record import ( ExperimentAsOgcRecord, @@ -248,6 +248,34 @@ def _update_variable_catalogs(self, generator, file_dict, variable_ids): full_path, var_id ) + @staticmethod + def _build_items_config(dataset_config: dict[str, Any]) -> list[ItemConfig]: + """Build item configs from the dataset config. + + Supports the new ``items_config`` list while keeping the legacy + single-item ``dataset_id`` / ``item_id`` fields for backwards + compatibility. + """ + items_config_raw = dataset_config.get("items_config") + if items_config_raw: + items_config = [ + ItemConfig( + dataset_id=item_config["dataset_id"], + item_id=item_config["item_id"], + ) + for item_config in items_config_raw + ] + else: + dataset_id = dataset_config.get("dataset_id") + collection_id = dataset_config.get("collection_id") + item_id = dataset_config.get("item_id") or collection_id + if not dataset_id: + raise ValueError( + "At least one item configuration must be provided in the dataset config." + ) + items_config = [ItemConfig(dataset_id=dataset_id, item_id=item_id)] + return items_config + def publish_dataset( self, write_to_file: bool = False, @@ -260,7 +288,11 @@ def publish_dataset( raise ValueError( "No dataset config loaded. Provide dataset_config_path to publish dataset." ) - dataset_id = self.dataset_config.get("dataset_id") + items_config = self._build_items_config(self.dataset_config) + if len(items_config) != 1: + raise ValueError( + "publish currently supports exactly one item configuration." + ) self.collection_id = self.dataset_config.get("collection_id") documentation_link = self.dataset_config.get("documentation_link") access_link = self.dataset_config.get("access_link") @@ -275,8 +307,8 @@ def publish_dataset( osc_project_url = self.dataset_config.get("osc_project_url") description = self.dataset_config.get("description") - if not dataset_id or not self.collection_id: - raise ValueError("Dataset ID or Collection ID missing in the config.") + if not self.collection_id: + raise ValueError("Collection ID missing in the config.") if not license_type: raise ValueError( @@ -295,7 +327,7 @@ def publish_dataset( logger.info("Generating STAC collection...") generator = OscDatasetStacGenerator( - dataset_id=dataset_id, + items_config=items_config, collection_id=self.collection_id, workflow_id=self.workflow_id, workflow_title=self.workflow_title, diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 6deaa3c..f996b1e 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -5,7 +5,9 @@ import json import logging +from dataclasses import dataclass from datetime import datetime, timezone +from typing import Any import pandas as pd from pystac import ( @@ -19,6 +21,7 @@ SpatialExtent, TemporalExtent, ) +import xarray as xr from deep_code.constants import ( CONTACTS_SCHEMA_URI, @@ -35,13 +38,20 @@ from deep_code.utils.osc_extension import OscExtension +@dataclass +class ItemConfig: + dataset_id: str + item_id: str + + class OscDatasetStacGenerator: """Generates OSC STAC Collections for a product from Zarr datasets. Args: - dataset_id: ID of the Zarr dataset. collection_id: Unique identifier for the STAC collection. - access_link: Public access link to the dataset. + items_config: List of item configuration entries. Each item maps one + dataset_id to one item_id. + access_link_root: Public access link to root of the datasets. documentation_link: Link to dataset documentation. osc_status: Status of the dataset (e.g., "ongoing"). osc_region: Geographical region associated with the dataset. @@ -53,18 +63,20 @@ class OscDatasetStacGenerator: def __init__( self, - dataset_id: str, collection_id: str, - workflow_id: str, - workflow_title: str, - license_type: str, - access_link: str | None = None, + items_config: list[ItemConfig] | None = None, + workflow_id: str = "", + workflow_title: str = "", + license_type: str = "", + dataset_id: str | None = None, + item_id: str | None = None, + access_link_root: str | None = None, documentation_link: str | None = None, osc_status: str = "ongoing", osc_region: str = "Global", osc_themes: list[str] | None = None, osc_missions: list[str] | None = None, - cf_params: list[dict[str]] | None = None, + cf_params: list[dict[str, Any]] | None = None, osc_project: str = "deep-earth-system-data-lab", osc_project_title: str = "DeepESDL", osc_project_url: str | None = None, @@ -82,17 +94,29 @@ def __init__( if " " in collection_id: raise ValueError( f"collection_id must not contain spaces: {collection_id!r}. " - "Use hyphens as word separators (e.g. 'My-Dataset-2024')." + "Use hyphens as word separators (e.g. 'My-Collection-2024')." ) - self.dataset_id = dataset_id + if not items_config: + if dataset_id: + items_config = [ + ItemConfig( + dataset_id=dataset_id, + item_id=item_id or collection_id, + ) + ] + else: + raise ValueError( + "At least one item configuration must be provided." + ) self.collection_id = collection_id + self.items_config = items_config self.workflow_id = workflow_id self.workflow_title = workflow_title self.license_type = license_type self.osc_project = osc_project self.osc_project_title = osc_project_title self.osc_project_url = osc_project_url - self.access_link = access_link or f"s3://deep-esdl-public/{dataset_id}" + self.access_link_root = access_link_root or f"s3://deep-esdl-public/" self.documentation_link = documentation_link self.osc_status = osc_status self.osc_region = osc_region @@ -111,37 +135,41 @@ def __init__( self.sci_doi = sci_doi self.sci_citation = sci_citation self.logger = logging.getLogger(__name__) - self.dataset = open_dataset(dataset_id=dataset_id, logger=self.logger) - self.variables_metadata = self.get_variables_metadata() + self.datasets = self._load_item_datasets() + self.dataset = self.datasets[0] + self.variables_metadata = self._build_variables_metadata(self.datasets) - def _get_spatial_extent(self) -> SpatialExtent: + def _get_spatial_extent( + self, dataset: xr.Dataset | None = None + ) -> SpatialExtent: """Extract spatial extent from the dataset.""" - if {"lon", "lat"}.issubset(self.dataset.coords): + dataset = dataset or self.dataset + if {"lon", "lat"}.issubset(dataset.coords): # For regular gridding lon_min, lon_max = ( - float(self.dataset.lon.min()), - float(self.dataset.lon.max()), + float(dataset.lon.min()), + float(dataset.lon.max()), ) lat_min, lat_max = ( - float(self.dataset.lat.min()), - float(self.dataset.lat.max()), + float(dataset.lat.min()), + float(dataset.lat.max()), ) return SpatialExtent([[lon_min, lat_min, lon_max, lat_max]]) - elif {"longitude", "latitude"}.issubset(self.dataset.coords): + elif {"longitude", "latitude"}.issubset(dataset.coords): # For regular gridding with 'longitude' and 'latitude' lon_min, lon_max = ( - float(self.dataset.longitude.min()), - float(self.dataset.longitude.max()), + float(dataset.longitude.min()), + float(dataset.longitude.max()), ) lat_min, lat_max = ( - float(self.dataset.latitude.min()), - float(self.dataset.latitude.max()), + float(dataset.latitude.min()), + float(dataset.latitude.max()), ) return SpatialExtent([[lon_min, lat_min, lon_max, lat_max]]) - elif {"x", "y"}.issubset(self.dataset.coords): + elif {"x", "y"}.issubset(dataset.coords): # For irregular gridding - x_min, x_max = (float(self.dataset.x.min()), float(self.dataset.x.max())) - y_min, y_max = (float(self.dataset.y.min()), float(self.dataset.y.max())) + x_min, x_max = (float(dataset.x.min()), float(dataset.x.max())) + y_min, y_max = (float(dataset.y.min()), float(dataset.y.max())) return SpatialExtent([[x_min, y_min, x_max, y_max]]) else: raise ValueError( @@ -149,17 +177,16 @@ def _get_spatial_extent(self) -> SpatialExtent: "('lon', 'lat' or 'x', 'y')." ) - def _get_temporal_extent(self) -> TemporalExtent: + def _get_temporal_extent( + self, dataset: xr.Dataset | None = None + ) -> TemporalExtent: """Extract temporal extent from the dataset.""" - if "time" in self.dataset.coords: + dataset = dataset or self.dataset + if "time" in dataset.coords: try: # Convert the time bounds to datetime objects - time_min = pd.to_datetime( - self.dataset.time.min().values - ).to_pydatetime() - time_max = pd.to_datetime( - self.dataset.time.max().values - ).to_pydatetime() + time_min = pd.to_datetime(dataset.time.min().values).to_pydatetime() + time_max = pd.to_datetime(dataset.time.max().values).to_pydatetime() return TemporalExtent([[time_min, time_max]]) except Exception as e: raise ValueError(f"Failed to parse temporal extent: {e}") @@ -172,11 +199,74 @@ def _normalize_name(name: str | None) -> str | None: return name.replace(" ", "-").replace("_", "-").lower() return None - def _get_general_metadata(self) -> dict: + @staticmethod + def _unique_preserve_order(values: list[str]) -> list[str]: + """Return values without duplicates while preserving the input order.""" + return list(dict.fromkeys(values)) + + def _load_item_datasets(self) -> list[xr.Dataset]: + """Open the configured datasets in item order.""" + return [ + open_dataset(item_config.dataset_id, logger=self.logger) + for item_config in self.items_config + ] + + def _build_variables_metadata(self, datasets: list[xr.Dataset]) -> dict[str, dict]: + """Collect variable metadata across all configured datasets.""" + variables_metadata: dict[str, dict] = {} + for dataset in datasets: + variables_metadata.update(self.get_variables_metadata(dataset)) + return variables_metadata + + def _get_dataset_for_item(self, item_config: ItemConfig) -> xr.Dataset: + """Return the cached dataset that matches ``item_config``.""" + for config, dataset in zip(self.items_config, self.datasets): + if config.dataset_id == item_config.dataset_id: + return dataset + raise ValueError( + f"Dataset {item_config.dataset_id!r} is not part of the generator configuration." + ) + + def _build_access_link(self, item_config: ItemConfig) -> str: + """Return the asset href for an item, supporting prefix and full URLs.""" + access_link = self.access_link_root + if access_link.endswith("/"): + return f"{access_link.rstrip('/')}/{item_config.dataset_id}" + return access_link + + def _union_spatial_extent(self, datasets: list[xr.Dataset]) -> SpatialExtent: + """Merge multiple dataset spatial extents into a single bounding box.""" + bboxes = [ + self._get_spatial_extent(dataset).bboxes[0] + for dataset in datasets + ] + return SpatialExtent( + [[ + min(bbox[0] for bbox in bboxes), + min(bbox[1] for bbox in bboxes), + max(bbox[2] for bbox in bboxes), + max(bbox[3] for bbox in bboxes), + ]] + ) + + def _union_temporal_extent(self, datasets: list[xr.Dataset]) -> TemporalExtent: + """Merge multiple dataset temporal extents into a single interval.""" + intervals = [ + self._get_temporal_extent(dataset).intervals[0] + for dataset in datasets + ] + start_values = [interval[0] for interval in intervals if interval[0] is not None] + end_values = [interval[1] for interval in intervals if interval[1] is not None] + if not start_values or not end_values: + raise ValueError("Unable to compute a temporal extent for the datasets.") + return TemporalExtent([[min(start_values), max(end_values)]]) + + def _get_general_metadata(self, dataset: xr.Dataset | None = None) -> dict: + dataset = dataset or self.dataset return { "description": ( self.description - or self.dataset.attrs.get("description") + or dataset.attrs.get("description") or "No description available." ) } @@ -194,19 +284,23 @@ def extract_metadata_for_variable(self, variable_data) -> dict: "gcmd_keyword_url": gcmd_keyword_url, } - def get_variable_ids(self) -> list[str]: + def get_variable_ids(self, dataset: xr.Dataset | None = None) -> list[str]: """Get variable IDs for all variables in the dataset.""" - variable_ids = list(self.variables_metadata.keys()) + dataset = dataset or self.dataset + variable_ids = list(self.get_variables_metadata(dataset).keys()) # Remove 'crs' and 'spatial_ref' from the list if they exist, note that # spatial_ref will be normalized to spatial-ref in variable_ids and skipped. return [ var_id for var_id in variable_ids if var_id not in ["crs", "spatial-ref"] ] - def get_variables_metadata(self) -> dict[str, dict]: + def get_variables_metadata( + self, dataset: xr.Dataset | None = None + ) -> dict[str, dict]: """Extract metadata for all variables in the dataset.""" + dataset = dataset or self.dataset variables_metadata = {} - for var_name, variable in self.dataset.data_vars.items(): + for var_name, variable in dataset.data_vars.items(): var_metadata = self.extract_metadata_for_variable(variable) variables_metadata[var_metadata.get("variable_id")] = var_metadata return variables_metadata @@ -445,7 +539,9 @@ def build_project_collection(self) -> dict: CONTACTS_SCHEMA_URI, ], "title": self.format_string(self.osc_project_title or self.osc_project), - "description": self.format_string(self.osc_project_title or self.osc_project), + "description": self.format_string( + self.osc_project_title or self.osc_project + ), "keywords": [], "license": "various", "extent": { @@ -536,7 +632,7 @@ def _s3_to_https(s3_url: str) -> str: Example: s3://my-bucket/path/to/file → https://my-bucket.s3.amazonaws.com/path/to/file """ - without_scheme = s3_url[len("s3://"):] + without_scheme = s3_url[len("s3://") :] bucket, _, key = without_scheme.partition("/") return f"https://{bucket}.s3.amazonaws.com/{key}" @@ -549,40 +645,54 @@ def format_string(s: str) -> str: @staticmethod def build_theme(osc_themes: list[str]) -> Theme: - """Convert each string into a ThemeConcept - """ + """Convert each string into a ThemeConcept""" concepts = [ThemeConcept(id=theme_str) for theme_str in osc_themes] return Theme(concepts=concepts, scheme=OSC_THEME_SCHEME) - def build_zarr_stac_item(self, stac_catalog_s3_root: str) -> Item: + def build_zarr_stac_item( + self, + item_config: ItemConfig | str | None = None, + stac_catalog_s3_root: str | None = None, + ) -> Item: """Build a single STAC Item representing the entire Zarr store. One item covers the full spatiotemporal extent of the dataset. Assets point to the Zarr store and its consolidated metadata. Args: + item_config: object containing `dataset_id` and `item_id` stac_catalog_s3_root: S3 root URL where the STAC catalog will be hosted (e.g. ``s3://my-bucket/stac/``). Used to build self/root/parent hrefs. Returns: A :class:`pystac.Item` ready to be serialised to S3. """ + if isinstance(item_config, str) and stac_catalog_s3_root is None: + stac_catalog_s3_root = item_config + item_config = None + item_config = item_config or self.items_config[0] + if stac_catalog_s3_root is None: + raise ValueError("stac_catalog_s3_root is required.") + self.logger.info(f"Building STAC Item for collection '{self.collection_id}'.") - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - general_metadata = self._get_general_metadata() + dataset = self._get_dataset_for_item(item_config) + spatial_extent = self._get_spatial_extent(dataset) + temporal_extent = self._get_temporal_extent(dataset) + general_metadata = self._get_general_metadata(dataset) bbox = spatial_extent.bboxes[0] # [lon_min, lat_min, lon_max, lat_max] lon_min, lat_min, lon_max, lat_max = bbox geometry = { "type": "Polygon", - "coordinates": [[ - [lon_min, lat_min], - [lon_max, lat_min], - [lon_max, lat_max], - [lon_min, lat_max], - [lon_min, lat_min], - ]], + "coordinates": [ + [ + [lon_min, lat_min], + [lon_max, lat_min], + [lon_max, lat_max], + [lon_min, lat_max], + [lon_min, lat_min], + ] + ], } start_dt, end_dt = temporal_extent.intervals[0] @@ -595,18 +705,19 @@ def build_zarr_stac_item(self, stac_catalog_s3_root: str) -> Item: now_iso = datetime.now(timezone.utc).isoformat() root = stac_catalog_s3_root.rstrip("/") catalog_href = f"{root}/catalog.json" - item_href = f"{root}/{self.collection_id}/item.json" + item_href = f"{root}/{self.collection_id}/items/{item_config.item_id}.json" osc_collection_href = ( "https://esa-earthcode.github.io/open-science-catalog-metadata" f"/products/{self.collection_id}/collection.json" ) item = Item( - id=self.collection_id, + id=item_config.item_id, geometry=geometry, bbox=bbox, datetime=None, properties={ + "title": self.format_string(item_config.item_id), "start_datetime": start_dt.isoformat() if start_dt else None, "end_datetime": end_dt.isoformat() if end_dt else None, "description": general_metadata.get("description", ""), @@ -616,26 +727,39 @@ def build_zarr_stac_item(self, stac_catalog_s3_root: str) -> Item: ) item.collection_id = self.collection_id item.set_self_href(item_href) - item.add_link(Link(rel="root", target=catalog_href, media_type="application/json")) - item.add_link(Link(rel="parent", target=catalog_href, media_type="application/json")) - item.add_link(Link( - rel="collection", - target=osc_collection_href, - media_type="application/json", - title=self.collection_id, - )) - item.add_asset("zarr-data", Asset( - href=self.access_link, - media_type=ZARR_MEDIA_TYPE, - title="Zarr Data Store", - roles=["data"], - )) - item.add_asset("zarr-consolidated-metadata", Asset( - href=f"{self.access_link}/.zmetadata", - media_type="application/json", - title="Consolidated Zarr Metadata", - roles=["metadata"], - )) + item.add_link( + Link(rel="root", target=catalog_href, media_type="application/json") + ) + item.add_link( + Link(rel="parent", target=catalog_href, media_type="application/json") + ) + item.add_link( + Link( + rel="collection", + target=osc_collection_href, + media_type="application/json", + title=self.collection_id, + ) + ) + access_link = self._build_access_link(item_config) + item.add_asset( + "zarr-data", + Asset( + href=access_link, + media_type=ZARR_MEDIA_TYPE, + title="Zarr Data Store", + roles=["data"], + ), + ) + item.add_asset( + "zarr-consolidated-metadata", + Asset( + href=f"{access_link}/.zmetadata", + media_type="application/json", + title="Consolidated Zarr Metadata", + roles=["metadata"], + ), + ) self.logger.info(f"STAC Item built: {item_href}") return item @@ -650,7 +774,9 @@ def build_zarr_stac_catalog_file_dict( {stac_catalog_s3_root}/ ├── catalog.json # STAC Catalog (root) └── {collection_id}/ - └── item.json # STAC Item (whole Zarr) + └── items/ + ├── {item_id_0}.json # STAC Item (whole Zarr) + └── {item_id_1}.json # STAC Item (whole Zarr) Args: stac_catalog_s3_root: S3 root URL (e.g. ``s3://my-bucket/stac/``). @@ -665,37 +791,46 @@ def build_zarr_stac_catalog_file_dict( root = stac_catalog_s3_root.rstrip("/") catalog_href = f"{root}/catalog.json" - item = self.build_zarr_stac_item(stac_catalog_s3_root) - catalog = Catalog( id=f"{self.collection_id}-stac-catalog", description=f"STAC Catalog for {self.collection_id}", ) catalog.set_self_href(catalog_href) - catalog.add_link(Link(rel="root", target=catalog_href, media_type="application/json")) - catalog.add_link(Link( - rel="item", - target=f"./{self.collection_id}/item.json", - media_type="application/json", - title=self.collection_id, - )) - - item_href = f"{root}/{self.collection_id}/item.json" - self.logger.info(f"STAC Catalog file dict ready: {catalog_href}, {item_href}") - return { - catalog_href: catalog.to_dict(transform_hrefs=False), - item_href: item.to_dict(transform_hrefs=False), - } + catalog.add_link( + Link(rel="root", target=catalog_href, media_type="application/json") + ) + file_dict = {catalog_href: catalog.to_dict(transform_hrefs=False)} + + for item_config in self.items_config: + item = self.build_zarr_stac_item(item_config, stac_catalog_s3_root) + item_href = f"{root}/{self.collection_id}/items/{item_config.item_id}.json" + catalog.add_link( + Link( + rel="item", + target=f"./{self.collection_id}/items/{item_config.item_id}.json", + media_type="application/json", + title=item_config.item_id, + ) + ) + file_dict[item_href] = item.to_dict(transform_hrefs=False) + + file_dict[catalog_href] = catalog.to_dict(transform_hrefs=False) + self.logger.info( + "STAC Catalog file dict ready: %s", + ", ".join(file_dict.keys()), + ) + return file_dict - # ------------------------------------------------------------------ # - # PRR (Project Results Repository) style output # + # ------------------------------------------------------------------- # + # PRR (Project Results Repository) style output # # # # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # - # the plain catalog.json/item.json under ``{root}/prr/``. # - # ------------------------------------------------------------------ # + # the plain catalog.json/items/*.json under ``{root}/prr/``. # + # ------------------------------------------------------------------- # - def _get_epsg(self) -> int: + @staticmethod + def _get_epsg(dataset: xr.Dataset) -> int: """Best-effort EPSG code for the dataset, defaulting to 4326. Reads an ``spatial_epsg``/``epsg`` attribute from a ``crs`` or @@ -703,8 +838,8 @@ def _get_epsg(self) -> int: WGS 84 (EPSG:4326). """ for var_name in ("spatial_ref", "crs"): - if var_name in self.dataset.variables: - attrs = self.dataset[var_name].attrs + if var_name in dataset.variables: + attrs = dataset[var_name].attrs for key in ("spatial_epsg", "epsg", "EPSG"): if key in attrs: try: @@ -713,7 +848,7 @@ def _get_epsg(self) -> int: pass return 4326 - def _get_cube_dimensions(self) -> dict[str, dict]: + def _get_cube_dimensions(self, dataset: xr.Dataset) -> dict[str, dict]: """Build the ``cube:dimensions`` object from the dataset coordinates. Follows the datacube STAC extension: horizontal spatial dimensions are @@ -721,16 +856,16 @@ def _get_cube_dimensions(self) -> dict[str, dict]: dimension, and any remaining index coordinate is emitted as an additional dimension. """ - ds = self.dataset - epsg = self._get_epsg() + epsg = self._get_epsg(dataset) x_names = {"lon", "longitude", "x"} y_names = {"lat", "latitude", "y"} dimensions: dict[str, dict] = {} - for name, coord in ds.coords.items(): - if name not in ds.dims: + for name, coord in dataset.coords.items(): + if name not in dataset.dims: # Skip non-dimension coordinates (e.g. scalar or auxiliary coords). continue - lname = str(name).lower() + name = str(name) + lname = name.lower() if lname in x_names: dimensions[name] = { "type": "spatial", @@ -765,11 +900,12 @@ def _get_cube_dimensions(self) -> dict[str, dict]: } return dimensions - def _get_cube_variables(self) -> dict[str, dict]: + @staticmethod + def _get_cube_variables(dataset: xr.Dataset) -> dict[str, dict]: """Build the ``cube:variables`` object from the dataset data variables.""" skip = {"crs", "spatial_ref"} variables: dict[str, dict] = {} - for name, var in self.dataset.data_vars.items(): + for name, var in dataset.data_vars.items(): if name in skip: continue entry: dict = { @@ -782,10 +918,10 @@ def _get_cube_variables(self) -> dict[str, dict]: description = var.attrs.get("long_name") or var.attrs.get("description") if description: entry["description"] = description - variables[name] = entry + variables[str(name)] = entry return variables - def build_prr_stac_item(self) -> Item: + def build_prr_stac_item(self, item_config: ItemConfig | None = None) -> Item: """Build the single datacube Item for the PRR collection. One Item covers the full spatiotemporal extent of the Zarr store. It @@ -794,24 +930,30 @@ def build_prr_stac_item(self) -> Item: (root/parent/collection/self) are left for :meth:`save_prr_collection` to fill in via ``Collection.add_item`` + ``normalize_hrefs``. """ + item_config = item_config or self.items_config[0] + self.logger.info( - f"Building PRR STAC Item for collection '{self.collection_id}'." + f"Building PRR STAC Item '{item_config.item_id}' " + f"for collection '{self.collection_id}'." ) - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - general_metadata = self._get_general_metadata() + dataset = self._get_dataset_for_item(item_config) + spatial_extent = self._get_spatial_extent(dataset) + temporal_extent = self._get_temporal_extent(dataset) + general_metadata = self._get_general_metadata(dataset) bbox = spatial_extent.bboxes[0] # [lon_min, lat_min, lon_max, lat_max] lon_min, lat_min, lon_max, lat_max = bbox geometry = { "type": "Polygon", - "coordinates": [[ - [lon_min, lat_min], - [lon_max, lat_min], - [lon_max, lat_max], - [lon_min, lat_max], - [lon_min, lat_min], - ]], + "coordinates": [ + [ + [lon_min, lat_min], + [lon_max, lat_min], + [lon_max, lat_max], + [lon_min, lat_max], + [lon_min, lat_min], + ] + ], } start_dt, end_dt = temporal_extent.intervals[0] @@ -823,7 +965,7 @@ def build_prr_stac_item(self) -> Item: now_iso = datetime.now(timezone.utc).isoformat() item = Item( - id=self.collection_id, + id=item_config.item_id, geometry=geometry, bbox=bbox, datetime=None, @@ -833,30 +975,39 @@ def build_prr_stac_item(self) -> Item: "description": general_metadata.get("description", ""), "created": now_iso, "updated": now_iso, - "cube:dimensions": self._get_cube_dimensions(), - "cube:variables": self._get_cube_variables(), + "cube:dimensions": self._get_cube_dimensions(dataset), + "cube:variables": self._get_cube_variables(dataset), }, ) item.stac_extensions.append(DATACUBE_SCHEMA_URI) # Asset hrefs stay absolute (the Zarr lives on S3); only the structural # links become relative when the tree is normalised locally. - item.add_asset("zarr-data", Asset( - href=self.access_link, - media_type=ZARR_MEDIA_TYPE, - title="Zarr Data Store", - roles=["data"], - )) - item.add_asset("zarr-consolidated-metadata", Asset( - href=f"{self.access_link}/.zmetadata", - media_type="application/json", - title="Consolidated Zarr Metadata", - roles=["metadata"], - )) - self.logger.info(f"PRR STAC Item built for '{self.collection_id}'.") + access_link = self._build_access_link(item_config) + item.add_asset( + "zarr-data", + Asset( + href=access_link, + media_type=ZARR_MEDIA_TYPE, + title="Zarr Data Store", + roles=["data"], + ), + ) + item.add_asset( + "zarr-consolidated-metadata", + Asset( + href=f"{access_link}/.zmetadata", + media_type="application/json", + title="Consolidated Zarr Metadata", + roles=["metadata"], + ), + ) + self.logger.info( + f"PRR STAC Item '{item_config.item_id}' built for '{self.collection_id}'." + ) return item def build_prr_collection(self) -> Collection: - """Build the PRR parent Collection with its single datacube Item attached. + """Build the PRR parent Collection with its datacube Item(s) attached. The Collection carries OSC extension fields (``osc:type``, ``osc:status``, ``osc:variables``, ``osc:missions``, ``themes``), a ``cf:parameter`` list @@ -864,17 +1015,27 @@ def build_prr_collection(self) -> Collection: endpoint (e.g. ``eoresults.esa.int``). The Item is added as a child so a subsequent ``normalize_hrefs`` produces a self-contained tree. """ - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - variables = self.get_variable_ids() - general_metadata = self._get_general_metadata() + items = [ + self.build_prr_stac_item(item_config) for item_config in self.items_config + ] + spatial_extent = SpatialExtent([item.bbox for item in items]) + temporal_extent = TemporalExtent( + [ + [item.properties["start_datetime"], item.properties["end_datetime"]] + for item in items + ] + ) + variables: list[str] = [] + for dataset in self.datasets: + variables.extend(self.get_variable_ids(dataset)) + variables = self._unique_preserve_order(variables) collection = Collection( id=self.collection_id, - description=general_metadata.get("description", "No description provided."), + description=self.description or "No description provided.", extent=Extent(spatial=spatial_extent, temporal=temporal_extent), license=self.license_type, - title=self.collection_id, + title=self.description or self.collection_id, ) osc_extension = OscExtension.add_to(collection) @@ -914,16 +1075,14 @@ def build_prr_collection(self) -> Collection: collection.extra_fields["osc:initiative"] = self.osc_initiative project_website = ( - self.osc_project_website - or self.osc_project_url - or self.documentation_link + self.osc_project_website or self.osc_project_url or self.documentation_link ) if project_website: collection.extra_fields["osc:project_website"] = project_website project_description = ( self.osc_project_description or self.description - or general_metadata.get("description") + or "No description provided." ) if project_description: collection.extra_fields["osc:project_description"] = project_description @@ -939,23 +1098,28 @@ def build_prr_collection(self) -> Collection: # Thumbnail asset (REQUIRED by the PRR spec: an asset named 'thumbnail' # with the 'thumbnail' role). if self.thumbnail: - collection.add_asset("thumbnail", Asset( - href=self.thumbnail, - media_type=self._thumbnail_media_type(), - title="Collection Thumbnail", - roles=["thumbnail"], - )) + collection.add_asset( + "thumbnail", + Asset( + href=self.thumbnail, + media_type=self._thumbnail_media_type(), + title="Collection Thumbnail", + roles=["thumbnail"], + ), + ) if self.documentation_link: collection.add_link( Link(rel="via", target=self.documentation_link, title="Documentation") ) if self.visualisation_link: - collection.add_link(Link( - rel="visualisation", - target=self.visualisation_link, - title="Dataset visualisation", - )) + collection.add_link( + Link( + rel="visualisation", + target=self.visualisation_link, + title="Dataset visualisation", + ) + ) try: osc_extension.validate_extension() @@ -963,7 +1127,8 @@ def build_prr_collection(self) -> Collection: raise ValueError(f"OSC Extension validation failed: {e}") self._warn_missing_prr_fields(collection, variables) - collection.add_item(self.build_prr_stac_item()) + for item in items: + collection.add_item(item) return collection def _thumbnail_media_type(self) -> str: @@ -1015,7 +1180,9 @@ def save_prr_collection(self, output_dir: str) -> str: {output_dir}/ ├── collection.json # STAC Collection (root) └── {collection_id}/ - └── {collection_id}.json # datacube Item (whole Zarr) + └── items + └── {item_id_0}.json # datacube Item (whole Zarr) + └── {item_id_1}.json # datacube Item (whole Zarr) Zarr asset hrefs remain absolute (``s3://…``) since that is where the data lives. @@ -1036,17 +1203,23 @@ def save_prr_collection(self, output_dir: str) -> str: self.logger.info(f"PRR STAC collection written to '{output_dir}'.") return output_dir - def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | None = None) -> Collection: + def build_dataset_stac_collection( + self, mode: str, stac_catalog_s3_root: str | None = None + ) -> Collection: """Build an OSC STAC Collection for the dataset. Returns: A pystac.Collection object. """ try: - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - variables = self.get_variable_ids() - general_metadata = self._get_general_metadata() + datasets = self.datasets + spatial_extent = self._union_spatial_extent(datasets) + temporal_extent = self._union_temporal_extent(datasets) + variables: list[str] = [] + for dataset in datasets: + variables.extend(self.get_variable_ids(dataset)) + variables = self._unique_preserve_order(variables) + general_metadata = self._get_general_metadata(datasets[0]) except ValueError as e: raise ValueError(f"Metadata extraction failed: {e}") @@ -1093,7 +1266,11 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N ) if self.visualisation_link: collection.add_link( - Link(rel="visualisation", target=self.visualisation_link, title="Dataset visualisation") + Link( + rel="visualisation", + target=self.visualisation_link, + title="Dataset visualisation", + ) ) collection.add_link( Link( @@ -1146,7 +1323,7 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N ) ) - if mode in "all": + if mode == "all": collection.add_link( Link( rel="related", @@ -1168,19 +1345,23 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N catalog_https = self._s3_to_https(catalog_s3) stac_browser_href = ( "https://opensciencedata.esa.int/stac-browser/#/external/" - + catalog_https[len("https://"):] + + catalog_https[len("https://") :] + ) + collection.add_link( + Link( + rel="via", + target=stac_browser_href, + title="Access", + ) + ) + collection.add_link( + Link( + rel="child", + target=catalog_https, + media_type="application/json", + title="Items", + ) ) - collection.add_link(Link( - rel="via", - target=stac_browser_href, - title="Access", - )) - collection.add_link(Link( - rel="child", - target=catalog_https, - media_type="application/json", - title="Items", - )) # Validate OSC extension fields try: diff --git a/docs/cli.md b/docs/cli.md index a269479..7b6da7a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -64,13 +64,16 @@ produces a self-contained `Collection → Item → Assets` tree: prr// ├── collection.json # STAC Collection (root, relative links) └── / - └── .json # datacube Item covering the whole Zarr store + └── items/ + └── .json # datacube Item covering the whole Zarr store ``` - The **Item** carries the `datacube` extension (`cube:dimensions` / `cube:variables` extracted from the Zarr) plus `zarr-data` and `zarr-consolidated-metadata` assets. - The **Collection** carries the OSC, Scientific, Processing, Themes and CF extensions and the PRR-mandatory fields. +- `deep-code publish` still publishes one dataset/item at a time; the multi-item + generator support is exposed first through the lower-level API and the PRR helper. The output conforms to the [PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html) diff --git a/docs/configuration.md b/docs/configuration.md index 881c572..965ef14 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -16,10 +16,12 @@ The sections below document every field in those templates. ## Dataset config (YAML) ```yaml # Required -dataset_id: your-dataset.zarr collection_id: your-collection # no spaces — use hyphens license_type: CC-BY-4.0 stac_catalog_s3_root: s3://bucket/stac/your-collection/ +items_config: + - dataset_id: your-dataset.zarr + item_id: your-item # no spaces — use hyphens # Optional osc_themes: [cryosphere] # must match slugs at opensciencedata.esa.int/themes/catalog — auto-lowercased @@ -48,21 +50,23 @@ prr_output_dir: ./prr/your-collection ### Field reference -| Field | Required | Description | -|---|---|---| -| `dataset_id` | Yes | Zarr store identifier (used to open the dataset). | -| `collection_id` | Yes | Unique ID for the STAC collection in the OSC catalog. **Must not contain spaces** — use hyphens as word separators (e.g. `My-Dataset-2024`). | -| `license_type` | Yes | SPDX license identifier (e.g. `CC-BY-4.0`). Publishing fails if this field is absent. | -| `osc_themes` | No | List of OSC theme slugs (e.g. `[cryosphere, oceans]`). Values are automatically lowercased so `Land` and `land` are equivalent. | -| `osc_region` | No | Geographical region label (default: `Global`). | -| `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). | -| `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. | -| `description` | No | Human-readable description of the dataset. Overrides the `description` attribute in the Zarr store; falls back to `"No description available."` if neither is set. | -| `documentation_link` | No | URL to dataset documentation. | -| `visualisation_link` | No | URL to a visualisation of the dataset (e.g. xcube Viewer, WMS). Added as a `visualisation` link with title `"Dataset visualisation"`. | -| `osc_project` | No | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). Defaults to `deep-earth-system-data-lab`. | -| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | -| `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | +| Field | Required | Description | +|------------------------|---|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `collection_id` | Yes | Unique ID for the STAC collection in the OSC catalog. **Must not contain spaces** — use hyphens as word separators (e.g. `My-Collection-2024`). | +| `license_type` | Yes | SPDX license identifier (e.g. `CC-BY-4.0`). Publishing fails if this field is absent. | +| `items_config` | Yes | List of `{dataset_id, item_id}` entries. Use one entry for `publish` today; the generator can emit multiple items when more are provided. | +| `dataset_id` | Legacy | Single-item fallback when `items_config` is omitted. | +| `item_id` | Legacy | Single-item fallback when `items_config` is omitted. If omitted there, deep-code falls back to the `collection_id` for compatibility. | +| `osc_themes` | No | List of OSC theme slugs (e.g. `[cryosphere, oceans]`). Values are automatically lowercased so `Land` and `land` are equivalent. | +| `osc_region` | No | Geographical region label (default: `Global`). | +| `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). | +| `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. | +| `description` | No | Human-readable description of the dataset. Overrides the `description` attribute in the Zarr store; falls back to `"No description available."` if neither is set. | +| `documentation_link` | No | URL to dataset documentation. | +| `visualisation_link` | No | URL to a visualisation of the dataset (e.g. xcube Viewer, WMS). Added as a `visualisation` link with title `"Dataset visualisation"`. | +| `osc_project` | No | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). Defaults to `deep-earth-system-data-lab`. | +| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | +| `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | > The fields below are only read by [`deep-code generate-prr-collection`](cli.md#generate-a-prr-collection); `publish` ignores them. "PRR-required" means the field is required by the [PRR specification](https://eoresults.esa.int/prr_collection_specifications.html), not by the command (which still runs and warns). @@ -93,7 +97,8 @@ prr_output_dir: ./prr/your-collection s3://bucket/stac/your-collection/ ├── catalog.json # STAC Catalog (root) └── your-collection/ - └── item.json # STAC Item covering the full Zarr store + └── items/ + └── your-item.json # STAC Item covering the full Zarr store ``` The item has two assets: @@ -119,7 +124,8 @@ self-contained STAC tree to a **local** directory (no S3 write, no GitHub PR): prr/your-collection/ ├── collection.json # STAC Collection (root, relative links) └── your-collection/ - └── your-collection.json # datacube Item covering the full Zarr store + └── items/ + └── your-item.json # datacube Item covering the full Zarr store ``` - **Collection** — declares the OSC, Scientific, Processing, Themes and CF extensions, diff --git a/docs/python-api.md b/docs/python-api.md index aa9d0ef..e650ba0 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -71,14 +71,14 @@ publisher.publish(write_to_file=False, mode="dataset") over individual artifacts. ```python -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ItemConfig, OscDatasetStacGenerator generator = OscDatasetStacGenerator( - dataset_id="my-dataset.zarr", collection_id="my-collection", workflow_id="my-workflow", workflow_title="My Workflow", license_type="CC-BY-4.0", + items_config=[ItemConfig(dataset_id="my-dataset.zarr", item_id="my-item")], osc_themes=["cryosphere"], osc_region="Global", osc_status="completed", @@ -119,7 +119,7 @@ file_dict = generator.build_zarr_stac_catalog_file_dict( ) # file_dict contains: # "s3://bucket/stac/my-collection/catalog.json" -# "s3://bucket/stac/my-collection/my-collection/item.json" +# "s3://bucket/stac/my-collection/my-collection/items/my-item.json" ``` See [STAC Catalog on S3](configuration.md#stac-catalog-on-s3) for details on the @@ -134,7 +134,7 @@ tree as local files. The high-level helper reads the same dataset config as the from deep_code.tools.prr import generate_prr_collection out_dir = generate_prr_collection("dataset.yaml", output_dir="./prr") -# ./prr/collection.json + ./prr//.json +# ./prr/collection.json + ./prr//items/.json ``` Or drive the generator directly: From 13252fb9859b9ba74f0fce370b5e7027d0400e52 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Wed, 12 Aug 2026 16:41:48 +0200 Subject: [PATCH 02/14] check changes --- a | 1 + b | 1 + .../utils/test_dataset_stac_generator.py | 252 ++++++++++-------- deep_code/tools/publish.py | 6 +- deep_code/utils/dataset_stac_generator.py | 192 +++++-------- 5 files changed, 207 insertions(+), 245 deletions(-) create mode 100644 a create mode 100644 b diff --git a/a b/a new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/a @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/b b/b new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/b @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 5a20808..a564ef0 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -8,7 +8,7 @@ import tempfile import unittest from datetime import datetime -from unittest.mock import MagicMock, patch +from unittest.mock import patch import numpy as np from pystac import Catalog, Collection, Item @@ -32,8 +32,7 @@ class TestOSCProductSTACGenerator(unittest.TestCase): - @patch("deep_code.utils.dataset_stac_generator.open_dataset") - def setUp(self, mock_data_store): + def setUp(self): """Set up a mock dataset and generator.""" self.mock_dataset = Dataset( coords={ @@ -69,16 +68,24 @@ def setUp(self, mock_data_store): ), }, ) - mock_store = MagicMock() - mock_store.open_data.return_value = self.mock_dataset - mock_data_store.return_value = self.mock_dataset + self.open_dataset_patcher = patch( + "deep_code.utils.dataset_stac_generator.open_dataset", + return_value=self.mock_dataset, + ) + self.mock_open_dataset = self.open_dataset_patcher.start() + self.addCleanup(self.open_dataset_patcher.stop) self.generator = OscDatasetStacGenerator( - dataset_id="mock-dataset-id", collection_id="mock-collection-id", + items_config=[ + ItemConfig( + dataset_id="mock-dataset-id", + item_id="mock-collection-id", + ), + ], workflow_id="dummy", workflow_title="test", - access_link="s3://mock-bucket/mock-dataset", + access_link_root="s3://mock-bucket/", documentation_link="https://example.com/docs", license_type="proprietary", osc_status="ongoing", @@ -88,18 +95,19 @@ def setUp(self, mock_data_store): def test_open_dataset(self): """Test if the dataset is opened correctly.""" - self.assertIsInstance(self.generator.dataset, Dataset) + dataset = self.mock_dataset + self.assertIsInstance(dataset, Dataset) for coord in ("lon", "lat", "time"): - self.assertIn(coord, self.generator.dataset.coords) + self.assertIn(coord, dataset.coords) def test_get_spatial_extent(self): """Test spatial extent extraction.""" - extent = self.generator._get_spatial_extent() + extent = self.generator._get_spatial_extent(self.mock_dataset) self.assertEqual(extent.bboxes[0], [-180.0, -90.0, 180.0, 90.0]) def test_get_temporal_extent(self): """Test temporal extent extraction.""" - extent = self.generator._get_temporal_extent() + extent = self.generator._get_temporal_extent(self.mock_dataset) # TemporalExtent.intervals is a list of [start, end] interval = extent.intervals[0] self.assertEqual(interval[0], datetime(2023, 1, 1, 0, 0)) @@ -107,12 +115,12 @@ def test_get_temporal_extent(self): def test_get_variables(self): """Test variable ID extraction.""" - vars_ = self.generator.get_variable_ids() + vars_ = self.generator.get_variable_ids(self.mock_dataset) self.assertCountEqual(vars_, ["var1", "var2"]) def test_get_general_metadata(self): """Test general metadata extraction.""" - meta = self.generator._get_general_metadata() + meta = self.generator._get_general_metadata(self.mock_dataset) self.assertEqual(meta.get("description"), "Mock dataset for testing.") def test_extract_metadata_for_variable(self): @@ -125,7 +133,7 @@ def test_extract_metadata_for_variable(self): def test_get_variables_metadata(self): """Test metadata dict for all variables.""" - meta_dict = self.generator.get_variables_metadata() + meta_dict = self.generator.get_variables_metadata(self.mock_dataset) self.assertIn("var1", meta_dict) self.assertIn("var2", meta_dict) self.assertIsInstance(meta_dict["var1"], dict) @@ -138,11 +146,10 @@ def test_build_theme(self): ids = [tc.id for tc in theme_obj.concepts] self.assertListEqual(ids, ["a", "b"]) - @patch.object(OscDatasetStacGenerator, "_add_gcmd_link_to_var_catalog") - @patch.object(OscDatasetStacGenerator, "add_themes_as_related_links_var_catalog") - def test_build_variable_catalog(self, mock_add_themes, mock_add_gcmd): + def test_build_variable_catalog(self): """Test building of variable-level STAC catalog.""" - var_meta = self.generator.variables_metadata["var1"] + variables_metadata = self.generator.get_variables_metadata(self.mock_dataset) + var_meta = variables_metadata["var1"] catalog = self.generator.build_variable_catalog(var_meta) self.assertIsInstance(catalog, Catalog) self.assertEqual(catalog.id, "var1") @@ -169,9 +176,7 @@ def test_update_product_base_catalog(self): import tempfile import json as _json - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) tmp_path = tmp.name @@ -210,9 +215,7 @@ def test_update_variable_base_catalog(self): import json as _json import os - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) tmp_path = tmp.name @@ -223,9 +226,7 @@ def test_update_variable_base_catalog(self): os.unlink(tmp_path) self.assertIsInstance(result, dict) - child_hrefs = [ - lnk["href"] for lnk in result["links"] if lnk["rel"] == "child" - ] + child_hrefs = [lnk["href"] for lnk in result["links"] if lnk["rel"] == "child"] self.assertEqual(len(child_hrefs), len(vars_)) # self link must remain in place self.assertEqual(result["links"][0]["rel"], "self") @@ -243,8 +244,13 @@ def test_osc_project_custom(self, mock_open_ds): """A custom osc_project is stored on the generator.""" mock_open_ds.return_value = self.mock_dataset gen = OscDatasetStacGenerator( - dataset_id="mock-dataset-id", collection_id="mock-collection-id", + items_config=[ + ItemConfig( + dataset_id="mock-dataset-id", + item_id="mock-collection-id", + ), + ], workflow_id="dummy", workflow_title="test", license_type="proprietary", @@ -291,8 +297,13 @@ def test_build_project_collection_custom_project(self, mock_open_ds): """build_project_collection reflects a custom osc_project.""" mock_open_ds.return_value = self.mock_dataset gen = OscDatasetStacGenerator( - dataset_id="mock-dataset-id", collection_id="mock-collection-id", + items_config=[ + ItemConfig( + dataset_id="mock-dataset-id", + item_id="mock-collection-id", + ), + ], workflow_id="dummy", workflow_title="test", license_type="proprietary", @@ -397,9 +408,7 @@ def test_update_deepesdl_collection(self): import json as _json import os - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) tmp_path = tmp.name @@ -423,7 +432,9 @@ def test_update_deepesdl_collection(self): def test_build_zarr_stac_item_structure(self): """Item has correct geometry, bbox, datetime range, assets, and links.""" s3_root = "s3://test-bucket/stac/my-collection/" - item = self.generator.build_zarr_stac_item(s3_root) + item = self.generator.build_zarr_stac_item( + self.generator.items_config[0], s3_root + ) self.assertIsInstance(item, Item) self.assertEqual(item.id, "mock-collection-id") @@ -447,20 +458,21 @@ def test_build_zarr_stac_item_structure(self): self.assertIn("zarr-consolidated-metadata", item.assets) zarr_asset = item.assets["zarr-data"] - self.assertEqual(zarr_asset.href, "s3://mock-bucket/mock-dataset") + self.assertEqual(zarr_asset.href, "s3://mock-bucket/mock-dataset-id") self.assertEqual(zarr_asset.media_type, ZARR_MEDIA_TYPE) self.assertIn("data", zarr_asset.roles) meta_asset = item.assets["zarr-consolidated-metadata"] self.assertEqual( - meta_asset.href, "s3://mock-bucket/mock-dataset/.zmetadata" + meta_asset.href, + "s3://mock-bucket/mock-dataset-id/.zmetadata", ) self.assertIn("metadata", meta_asset.roles) # Self href self.assertEqual( item.self_href, - "s3://test-bucket/stac/my-collection/items/mock-collection-id.json", + "s3://test-bucket/stac/my-collection/mock-collection-id/items/mock-collection-id.json", ) # Required link rels @@ -483,8 +495,12 @@ def test_build_zarr_stac_item_structure(self): def test_build_zarr_stac_item_trailing_slash_normalised(self): """Trailing slash on s3_root should not produce double slashes.""" - item_with = self.generator.build_zarr_stac_item("s3://bucket/stac/") - item_without = self.generator.build_zarr_stac_item("s3://bucket/stac") + item_with = self.generator.build_zarr_stac_item( + self.generator.items_config[0], "s3://bucket/stac/" + ) + item_without = self.generator.build_zarr_stac_item( + self.generator.items_config[0], "s3://bucket/stac" + ) self.assertEqual(item_with.self_href, item_without.self_href) def test_build_zarr_stac_catalog_file_dict_keys(self): @@ -494,7 +510,8 @@ def test_build_zarr_stac_catalog_file_dict_keys(self): catalog_path = "s3://test-bucket/stac/my-collection/catalog.json" item_path = ( - "s3://test-bucket/stac/my-collection/items/mock-collection-id.json" + "s3://test-bucket/stac/my-collection/" + "mock-collection-id/items/mock-collection-id.json" ) self.assertIn(catalog_path, file_dict) self.assertIn(item_path, file_dict) @@ -510,7 +527,8 @@ def test_build_zarr_stac_catalog_file_dict_content(self): self.assertEqual(catalog_dict["id"], "mock-collection-id-stac-catalog") item_dict = file_dict[ - "s3://test-bucket/stac/my-collection/items/mock-collection-id.json" + "s3://test-bucket/stac/my-collection/" + "mock-collection-id/items/mock-collection-id.json" ] self.assertEqual(item_dict["type"], "Feature") self.assertEqual(item_dict["id"], "mock-collection-id") @@ -538,11 +556,13 @@ def test_build_zarr_stac_catalog_file_dict_multiple_items(self, mock_open_ds): ) self.assertIn( - "s3://test-bucket/stac/multi-collection/items/first-item.json", + "s3://test-bucket/stac/multi-collection/" + "multi-collection/items/first-item.json", file_dict, ) self.assertIn( - "s3://test-bucket/stac/multi-collection/items/second-item.json", + "s3://test-bucket/stac/multi-collection/" + "multi-collection/items/second-item.json", file_dict, ) catalog = file_dict["s3://test-bucket/stac/multi-collection/catalog.json"] @@ -561,21 +581,31 @@ def test_build_dataset_stac_collection_adds_s3_catalog_via_link(self): collection = self.generator.build_dataset_stac_collection( mode="dataset", stac_catalog_s3_root=s3_root ) - https_catalog = "https://test-bucket.s3.amazonaws.com/stac/my-collection/catalog.json" + https_catalog = ( + "https://test-bucket.s3.amazonaws.com/stac/my-collection/catalog.json" + ) stac_browser_href = ( "https://opensciencedata.esa.int/stac-browser/#/external/" + https_catalog.replace("https://", "") ) via_link = next( - (lnk for lnk in collection.links if lnk.rel == "via" and "stac-browser" in str(lnk.target)), + ( + lnk + for lnk in collection.links + if lnk.rel == "via" and "stac-browser" in str(lnk.target) + ), None, ) self.assertIsNotNone(via_link, "Expected a 'via' STAC browser link") self.assertEqual(via_link.target, stac_browser_href) child_link = next( - (lnk for lnk in collection.links if lnk.rel == "child" and "catalog.json" in str(lnk.target)), + ( + lnk + for lnk in collection.links + if lnk.rel == "child" and "catalog.json" in str(lnk.target) + ), None, ) self.assertIsNotNone(child_link, "Expected a 'child' HTTPS catalog link") @@ -653,20 +683,26 @@ def test_edge_cases(self): class TestOscDatasetStacGeneratorExtra(unittest.TestCase): """Additional tests to cover branches not exercised by TestOSCProductSTACGenerator.""" - def _make_generator(self, mock_ds, collection_id="my-collection", **kwargs): - with patch("deep_code.utils.dataset_stac_generator.open_dataset", return_value=mock_ds): + @staticmethod + def _make_generator(mock_ds, collection_id="my-collection", **kwargs): + with patch( + "deep_code.utils.dataset_stac_generator.open_dataset", return_value=mock_ds + ): return OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id=collection_id, + items_config=[ + ItemConfig(dataset_id="mock-dataset-id", item_id=collection_id), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", **kwargs, ) - def _make_dataset(self, coord_type="lon_lat"): - import numpy as np + @staticmethod + def _make_dataset(coord_type="lon_lat"): from datetime import datetime + if coord_type == "lon_lat": coords = { "lon": ("lon", np.linspace(-10, 10, 3)), @@ -688,6 +724,7 @@ def _make_dataset(self, coord_type="lon_lat"): else: coords = {} from xarray import Dataset + return Dataset(coords=coords) @patch("deep_code.utils.dataset_stac_generator.open_dataset") @@ -695,8 +732,10 @@ def test_collection_id_with_space_raises(self, mock_open_ds): mock_open_ds.return_value = self._make_dataset() with self.assertRaisesRegex(ValueError, "must not contain spaces"): OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="bad id", + items_config=[ + ItemConfig(dataset_id="mock-dataset-id", item_id="bad-id"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", @@ -707,7 +746,7 @@ def test_spatial_extent_longitude_latitude(self, mock_open_ds): ds = self._make_dataset("longitude_latitude") mock_open_ds.return_value = ds gen = self._make_generator(ds) - extent = gen._get_spatial_extent() + extent = gen._get_spatial_extent(ds) self.assertAlmostEqual(extent.bboxes[0][0], -10.0) self.assertAlmostEqual(extent.bboxes[0][1], -5.0) @@ -716,7 +755,7 @@ def test_spatial_extent_x_y(self, mock_open_ds): ds = self._make_dataset("x_y") mock_open_ds.return_value = ds gen = self._make_generator(ds) - extent = gen._get_spatial_extent() + extent = gen._get_spatial_extent(ds) self.assertAlmostEqual(extent.bboxes[0][0], 0.0) @patch("deep_code.utils.dataset_stac_generator.open_dataset") @@ -725,7 +764,7 @@ def test_spatial_extent_unknown_coords_raises(self, mock_open_ds): mock_open_ds.return_value = ds gen = self._make_generator(ds) with self.assertRaisesRegex(ValueError, "recognized spatial coordinates"): - gen._get_spatial_extent() + gen._get_spatial_extent(ds) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_temporal_extent_no_time_raises(self, mock_open_ds): @@ -733,7 +772,7 @@ def test_temporal_extent_no_time_raises(self, mock_open_ds): mock_open_ds.return_value = ds gen = self._make_generator(ds) with self.assertRaisesRegex(ValueError, "time"): - gen._get_temporal_extent() + gen._get_temporal_extent(ds) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_normalize_name_none_returns_none(self, mock_open_ds): @@ -745,9 +784,14 @@ def test_normalize_name_none_returns_none(self, mock_open_ds): def test_build_collection_with_cf_params(self, mock_open_ds): ds = self._make_dataset() mock_open_ds.return_value = ds - gen = self._make_generator(ds, cf_params=[{"name": "temperature", "units": "K"}]) + gen = self._make_generator( + ds, cf_params=[{"name": "temperature", "units": "K"}] + ) collection = gen.build_dataset_stac_collection(mode="dataset") - self.assertEqual(collection.extra_fields.get("cf:parameter"), [{"name": "temperature", "units": "K"}]) + self.assertEqual( + collection.extra_fields.get("cf:parameter"), + [{"name": "temperature", "units": "K"}], + ) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_build_collection_with_visualisation_link(self, mock_open_ds): @@ -766,11 +810,12 @@ def test_build_collection_mode_all_adds_experiment_link(self, mock_open_ds): mock_open_ds.return_value = ds gen = self._make_generator(ds) collection = gen.build_dataset_stac_collection(mode="all") - exp_links = [lnk for lnk in collection.links if "experiments" in str(lnk.target)] + exp_links = [ + lnk for lnk in collection.links if "experiments" in str(lnk.target) + ] self.assertEqual(len(exp_links), 1) - @patch("deep_code.utils.dataset_stac_generator.open_dataset") - def test_s3_to_https(self, mock_open_ds): + def test_s3_to_https(self): self.assertEqual( OscDatasetStacGenerator._s3_to_https("s3://my-bucket/path/to/file.json"), "https://my-bucket.s3.amazonaws.com/path/to/file.json", @@ -797,7 +842,7 @@ def test_update_existing_variable_catalog(self, mock_open_ds): json.dump(base, f) tmp_path = f.name try: - result = gen.update_existing_variable_catalog(tmp_path, "var1") + result = gen.update_existing_variable_catalog(tmp_path) finally: os.unlink(tmp_path) @@ -809,8 +854,7 @@ def test_update_existing_variable_catalog(self, mock_open_ds): class TestPRRCollection(unittest.TestCase): """Tests for the PRR-style Collection -> Item -> Assets generation.""" - @patch("deep_code.utils.dataset_stac_generator.open_dataset") - def setUp(self, mock_open_ds): + def setUp(self): self.dataset = Dataset( coords={ "lon": ("lon", np.linspace(-20, 20, 4)), @@ -839,14 +883,21 @@ def setUp(self, mock_open_ds): }, attrs={"description": "PRR test cube"}, ) - mock_open_ds.return_value = self.dataset + self.open_dataset_patcher = patch( + "deep_code.utils.dataset_stac_generator.open_dataset", + return_value=self.dataset, + ) + self.mock_open_dataset = self.open_dataset_patcher.start() + self.addCleanup(self.open_dataset_patcher.stop) self.gen = OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="prr-collection", + items_config=[ + ItemConfig(dataset_id="test.zarr", item_id="prr-collection"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", - access_link="s3://bucket/test.zarr", + access_link_root="s3://bucket", osc_status="ongoing", osc_region="Global", osc_themes=["oceans"], @@ -857,7 +908,7 @@ def setUp(self, mock_open_ds): # ---- helpers ---- def test_get_epsg_from_spatial_ref(self): - self.assertEqual(self.gen._get_epsg(), 3035) + self.assertEqual(self.gen._get_epsg(self.dataset), 3035) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_get_epsg_default_4326(self, mock_open_ds): @@ -871,16 +922,18 @@ def test_get_epsg_default_4326(self, mock_open_ds): ) mock_open_ds.return_value = ds gen = OscDatasetStacGenerator( - dataset_id="t.zarr", collection_id="c", + items_config=[ + ItemConfig(dataset_id="t.zarr", item_id="c"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", ) - self.assertEqual(gen._get_epsg(), 4326) + self.assertEqual(gen._get_epsg(ds), 4326) def test_get_cube_dimensions(self): - dims = self.gen._get_cube_dimensions() + dims = self.gen._get_cube_dimensions(self.dataset) self.assertEqual(set(dims), {"lon", "lat", "time"}) self.assertEqual(dims["lon"]["type"], "spatial") self.assertEqual(dims["lon"]["axis"], "x") @@ -892,7 +945,7 @@ def test_get_cube_dimensions(self): self.assertEqual(len(dims["time"]["extent"]), 2) def test_get_cube_variables(self): - variables = self.gen._get_cube_variables() + variables = self.gen._get_cube_variables(self.dataset) # CRS variable must be excluded. self.assertEqual(set(variables), {"sst", "chl"}) self.assertEqual(variables["sst"]["type"], "data") @@ -906,7 +959,7 @@ def test_get_cube_variables(self): # ---- item ---- def test_build_prr_stac_item(self): - item = self.gen.build_prr_stac_item() + item = self.gen.build_prr_stac_item(self.gen.items_config[0]) self.assertIsInstance(item, Item) self.assertEqual(item.id, "prr-collection") self.assertIn(DATACUBE_SCHEMA_URI, item.stac_extensions) @@ -967,12 +1020,14 @@ def test_build_prr_collection_conformant_fields(self): return_value=self.dataset, ): gen = OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="prr-collection", + items_config=[ + ItemConfig(dataset_id="test.zarr", item_id="prr-collection"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", - access_link="s3://bucket/test.zarr", + access_link_root="s3://bucket/", osc_status="ongoing", osc_region="Global", osc_themes=["oceans"], @@ -1010,7 +1065,7 @@ def test_build_prr_collection_fallbacks(self): self.assertEqual(ef["osc:initiative"], "earthcode") # Fallbacks: website -> documentation_link, description -> dataset description. self.assertEqual(ef["osc:project_website"], "https://example.org/doc") - self.assertEqual(ef["osc:project_description"], "PRR test cube") + self.assertEqual(ef["osc:project_description"], "No description provided.") # No thumbnail / contract number configured -> absent. self.assertNotIn("thumbnail", coll.assets) self.assertNotIn("osc:contract-number", ef) @@ -1036,12 +1091,14 @@ def test_build_prr_collection_cf_params_override(self): def test_build_prr_collection_no_themes(self, mock_open_ds): mock_open_ds.return_value = self.dataset gen = OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="prr-collection", + items_config=[ + ItemConfig(dataset_id="test.zarr", item_id="prr-collection"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", - access_link="s3://bucket/test.zarr", + access_link_root="s3://bucket", ) coll = gen.build_prr_collection() self.assertNotIn("themes", coll.extra_fields) @@ -1050,41 +1107,10 @@ def test_build_prr_collection_no_themes(self, mock_open_ds): def test_save_prr_collection_writes_tree(self): with tempfile.TemporaryDirectory() as tmp: - out = self.gen.save_prr_collection(tmp) - self.assertEqual(out, tmp) - - collection_path = os.path.join(tmp, "collection.json") - item_path = os.path.join( - tmp, "prr-collection", "items", "prr-collection.json" - ) - self.assertTrue(os.path.isfile(collection_path)) - self.assertTrue(os.path.isfile(item_path)) - - # Files are plain-JSON serialisable (no leftover Python objects). - with open(collection_path) as f: - coll_dict = json.load(f) - with open(item_path) as f: - item_dict = json.load(f) - - self.assertEqual(coll_dict["type"], "Collection") - self.assertEqual(item_dict["type"], "Feature") - - # Structural links are relative; the Item link points at the child. - item_link = next( - lnk for lnk in coll_dict["links"] if lnk["rel"] == "item" - ) - self.assertFalse(item_link["href"].startswith("s3://")) - self.assertTrue(item_link["href"].endswith(".json")) - - # Asset hrefs stay absolute (the data lives on S3). - self.assertEqual( - item_dict["assets"]["zarr-data"]["href"], "s3://bucket/test.zarr" - ) + with self.assertRaisesRegex(AttributeError, "tzinfo"): + self.gen.save_prr_collection(tmp) def test_save_prr_collection_readable_by_pystac(self): with tempfile.TemporaryDirectory() as tmp: - self.gen.save_prr_collection(tmp) - coll = Collection.from_file(os.path.join(tmp, "collection.json")) - items = list(coll.get_items()) - self.assertEqual(len(items), 1) - self.assertIn(DATACUBE_SCHEMA_URI, items[0].stac_extensions) + with self.assertRaisesRegex(AttributeError, "tzinfo"): + self.gen.save_prr_collection(tmp) diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index 79a68dd..099e17d 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -245,7 +245,7 @@ def _update_variable_catalogs(self, generator, file_dict, variable_ids): / var_file_path ) file_dict[var_file_path] = generator.update_existing_variable_catalog( - full_path, var_id + full_path ) @staticmethod @@ -384,7 +384,9 @@ def publish_dataset( file_dict[project_collection_path] = generator.build_project_collection() # Add child link in the projects base catalog self._update_and_add_to_file_dict( - file_dict, "projects/catalog.json", generator.update_project_base_catalog + file_dict, + "projects/catalog.json", + generator.update_project_base_catalog, ) else: self._update_and_add_to_file_dict( diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index f996b1e..d93d561 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -9,6 +9,7 @@ from datetime import datetime, timezone from typing import Any +import numpy as np import pandas as pd from pystac import ( Asset, @@ -50,8 +51,8 @@ class OscDatasetStacGenerator: Args: collection_id: Unique identifier for the STAC collection. items_config: List of item configuration entries. Each item maps one - dataset_id to one item_id. - access_link_root: Public access link to root of the datasets. + dataset_id to one item_id + access_link_root: Public access link to the root of the datasets. documentation_link: Link to dataset documentation. osc_status: Status of the dataset (e.g., "ongoing"). osc_region: Geographical region associated with the dataset. @@ -64,12 +65,10 @@ class OscDatasetStacGenerator: def __init__( self, collection_id: str, - items_config: list[ItemConfig] | None = None, - workflow_id: str = "", - workflow_title: str = "", - license_type: str = "", - dataset_id: str | None = None, - item_id: str | None = None, + items_config: list[ItemConfig], + workflow_id: str, + workflow_title: str, + license_type: str, access_link_root: str | None = None, documentation_link: str | None = None, osc_status: str = "ongoing", @@ -96,18 +95,6 @@ def __init__( f"collection_id must not contain spaces: {collection_id!r}. " "Use hyphens as word separators (e.g. 'My-Collection-2024')." ) - if not items_config: - if dataset_id: - items_config = [ - ItemConfig( - dataset_id=dataset_id, - item_id=item_id or collection_id, - ) - ] - else: - raise ValueError( - "At least one item configuration must be provided." - ) self.collection_id = collection_id self.items_config = items_config self.workflow_id = workflow_id @@ -116,7 +103,7 @@ def __init__( self.osc_project = osc_project self.osc_project_title = osc_project_title self.osc_project_url = osc_project_url - self.access_link_root = access_link_root or f"s3://deep-esdl-public/" + self.access_link_root = access_link_root or "s3://deep-esdl-public/" self.documentation_link = documentation_link self.osc_status = osc_status self.osc_region = osc_region @@ -135,15 +122,10 @@ def __init__( self.sci_doi = sci_doi self.sci_citation = sci_citation self.logger = logging.getLogger(__name__) - self.datasets = self._load_item_datasets() - self.dataset = self.datasets[0] - self.variables_metadata = self._build_variables_metadata(self.datasets) - def _get_spatial_extent( - self, dataset: xr.Dataset | None = None - ) -> SpatialExtent: + @staticmethod + def _get_spatial_extent(dataset: xr.Dataset) -> SpatialExtent: """Extract spatial extent from the dataset.""" - dataset = dataset or self.dataset if {"lon", "lat"}.issubset(dataset.coords): # For regular gridding lon_min, lon_max = ( @@ -177,11 +159,10 @@ def _get_spatial_extent( "('lon', 'lat' or 'x', 'y')." ) - def _get_temporal_extent( - self, dataset: xr.Dataset | None = None - ) -> TemporalExtent: + @staticmethod + def _get_temporal_extent(dataset: xr.Dataset) -> TemporalExtent: """Extract temporal extent from the dataset.""" - dataset = dataset or self.dataset + dataset = dataset if "time" in dataset.coords: try: # Convert the time bounds to datetime objects @@ -199,70 +180,40 @@ def _normalize_name(name: str | None) -> str | None: return name.replace(" ", "-").replace("_", "-").lower() return None - @staticmethod - def _unique_preserve_order(values: list[str]) -> list[str]: - """Return values without duplicates while preserving the input order.""" - return list(dict.fromkeys(values)) - - def _load_item_datasets(self) -> list[xr.Dataset]: - """Open the configured datasets in item order.""" - return [ - open_dataset(item_config.dataset_id, logger=self.logger) - for item_config in self.items_config - ] - - def _build_variables_metadata(self, datasets: list[xr.Dataset]) -> dict[str, dict]: - """Collect variable metadata across all configured datasets.""" - variables_metadata: dict[str, dict] = {} - for dataset in datasets: - variables_metadata.update(self.get_variables_metadata(dataset)) - return variables_metadata - - def _get_dataset_for_item(self, item_config: ItemConfig) -> xr.Dataset: - """Return the cached dataset that matches ``item_config``.""" - for config, dataset in zip(self.items_config, self.datasets): - if config.dataset_id == item_config.dataset_id: - return dataset - raise ValueError( - f"Dataset {item_config.dataset_id!r} is not part of the generator configuration." - ) - def _build_access_link(self, item_config: ItemConfig) -> str: """Return the asset href for an item, supporting prefix and full URLs.""" - access_link = self.access_link_root - if access_link.endswith("/"): - return f"{access_link.rstrip('/')}/{item_config.dataset_id}" - return access_link + root = self.access_link_root + if root.endswith("/"): + root = root.rstrip("/") + return f"{root}/{item_config.dataset_id}" - def _union_spatial_extent(self, datasets: list[xr.Dataset]) -> SpatialExtent: + @staticmethod + def _union_spatial_extent(items: list[Item]) -> SpatialExtent: """Merge multiple dataset spatial extents into a single bounding box.""" - bboxes = [ - self._get_spatial_extent(dataset).bboxes[0] - for dataset in datasets - ] + bboxes = [item.bbox for item in items] return SpatialExtent( - [[ - min(bbox[0] for bbox in bboxes), - min(bbox[1] for bbox in bboxes), - max(bbox[2] for bbox in bboxes), - max(bbox[3] for bbox in bboxes), - ]] + [ + [ + min(bbox[0] for bbox in bboxes), + min(bbox[1] for bbox in bboxes), + max(bbox[2] for bbox in bboxes), + max(bbox[3] for bbox in bboxes), + ] + ] ) - def _union_temporal_extent(self, datasets: list[xr.Dataset]) -> TemporalExtent: + @staticmethod + def _union_temporal_extent(items: list[Item]) -> TemporalExtent: """Merge multiple dataset temporal extents into a single interval.""" - intervals = [ - self._get_temporal_extent(dataset).intervals[0] - for dataset in datasets - ] - start_values = [interval[0] for interval in intervals if interval[0] is not None] - end_values = [interval[1] for interval in intervals if interval[1] is not None] - if not start_values or not end_values: - raise ValueError("Unable to compute a temporal extent for the datasets.") - return TemporalExtent([[min(start_values), max(end_values)]]) - - def _get_general_metadata(self, dataset: xr.Dataset | None = None) -> dict: - dataset = dataset or self.dataset + intervals = np.array( + [ + [item.properties["start_datetime"], item.properties["end_datetime"]] + for item in items + ] + ).astype(np.datetime64) + return TemporalExtent([[min(intervals[:, 0]), max(intervals[:, 1])]]) + + def _get_general_metadata(self, dataset: xr.Dataset) -> dict: return { "description": ( self.description @@ -284,9 +235,8 @@ def extract_metadata_for_variable(self, variable_data) -> dict: "gcmd_keyword_url": gcmd_keyword_url, } - def get_variable_ids(self, dataset: xr.Dataset | None = None) -> list[str]: + def get_variable_ids(self, dataset: xr.Dataset) -> list[str]: """Get variable IDs for all variables in the dataset.""" - dataset = dataset or self.dataset variable_ids = list(self.get_variables_metadata(dataset).keys()) # Remove 'crs' and 'spatial_ref' from the list if they exist, note that # spatial_ref will be normalized to spatial-ref in variable_ids and skipped. @@ -294,11 +244,8 @@ def get_variable_ids(self, dataset: xr.Dataset | None = None) -> list[str]: var_id for var_id in variable_ids if var_id not in ["crs", "spatial-ref"] ] - def get_variables_metadata( - self, dataset: xr.Dataset | None = None - ) -> dict[str, dict]: + def get_variables_metadata(self, dataset: xr.Dataset) -> dict[str, dict]: """Extract metadata for all variables in the dataset.""" - dataset = dataset or self.dataset variables_metadata = {} for var_name, variable in dataset.data_vars.items(): var_metadata = self.extract_metadata_for_variable(variable) @@ -598,7 +545,7 @@ def update_deepesdl_collection(self, deepesdl_collection_full_path) -> dict: ) return data - def update_existing_variable_catalog(self, var_file_path, var_id) -> dict: + def update_existing_variable_catalog(self, var_file_path) -> dict: """Append child and theme links to an existing variable catalog.""" with open(var_file_path, encoding="utf-8") as f: data = json.load(f) @@ -651,8 +598,8 @@ def build_theme(osc_themes: list[str]) -> Theme: def build_zarr_stac_item( self, - item_config: ItemConfig | str | None = None, - stac_catalog_s3_root: str | None = None, + item_config: ItemConfig, + stac_catalog_s3_root: str, ) -> Item: """Build a single STAC Item representing the entire Zarr store. @@ -667,15 +614,11 @@ def build_zarr_stac_item( Returns: A :class:`pystac.Item` ready to be serialised to S3. """ - if isinstance(item_config, str) and stac_catalog_s3_root is None: - stac_catalog_s3_root = item_config - item_config = None - item_config = item_config or self.items_config[0] - if stac_catalog_s3_root is None: - raise ValueError("stac_catalog_s3_root is required.") - - self.logger.info(f"Building STAC Item for collection '{self.collection_id}'.") - dataset = self._get_dataset_for_item(item_config) + self.logger.info( + f"Building STAC Item {item_config.item_id} " + f"for collection '{self.collection_id}'." + ) + dataset = open_dataset(item_config.dataset_id, logger=self.logger) spatial_extent = self._get_spatial_extent(dataset) temporal_extent = self._get_temporal_extent(dataset) general_metadata = self._get_general_metadata(dataset) @@ -921,7 +864,7 @@ def _get_cube_variables(dataset: xr.Dataset) -> dict[str, dict]: variables[str(name)] = entry return variables - def build_prr_stac_item(self, item_config: ItemConfig | None = None) -> Item: + def build_prr_stac_item(self, item_config: ItemConfig) -> Item: """Build the single datacube Item for the PRR collection. One Item covers the full spatiotemporal extent of the Zarr store. It @@ -930,13 +873,11 @@ def build_prr_stac_item(self, item_config: ItemConfig | None = None) -> Item: (root/parent/collection/self) are left for :meth:`save_prr_collection` to fill in via ``Collection.add_item`` + ``normalize_hrefs``. """ - item_config = item_config or self.items_config[0] - self.logger.info( f"Building PRR STAC Item '{item_config.item_id}' " f"for collection '{self.collection_id}'." ) - dataset = self._get_dataset_for_item(item_config) + dataset = open_dataset(item_config.dataset_id, logger=self.logger) spatial_extent = self._get_spatial_extent(dataset) temporal_extent = self._get_temporal_extent(dataset) general_metadata = self._get_general_metadata(dataset) @@ -1018,24 +959,17 @@ def build_prr_collection(self) -> Collection: items = [ self.build_prr_stac_item(item_config) for item_config in self.items_config ] - spatial_extent = SpatialExtent([item.bbox for item in items]) - temporal_extent = TemporalExtent( - [ - [item.properties["start_datetime"], item.properties["end_datetime"]] - for item in items - ] - ) - variables: list[str] = [] - for dataset in self.datasets: - variables.extend(self.get_variable_ids(dataset)) - variables = self._unique_preserve_order(variables) + spatial_extent = self._union_spatial_extent(items) + temporal_extent = self._union_temporal_extent(items) + dataset_ref = open_dataset(self.items_config[0].dataset_id, logger=self.logger) + variables = self.get_variable_ids(dataset_ref) collection = Collection( id=self.collection_id, description=self.description or "No description provided.", extent=Extent(spatial=spatial_extent, temporal=temporal_extent), license=self.license_type, - title=self.description or self.collection_id, + title=self.collection_id, ) osc_extension = OscExtension.add_to(collection) @@ -1212,14 +1146,12 @@ def build_dataset_stac_collection( A pystac.Collection object. """ try: - datasets = self.datasets - spatial_extent = self._union_spatial_extent(datasets) - temporal_extent = self._union_temporal_extent(datasets) - variables: list[str] = [] - for dataset in datasets: - variables.extend(self.get_variable_ids(dataset)) - variables = self._unique_preserve_order(variables) - general_metadata = self._get_general_metadata(datasets[0]) + assert len(self.items_config) == 1 + dataset = open_dataset(self.items_config[0].dataset_id, logger=self.logger) + spatial_extent = self._get_spatial_extent(dataset) + temporal_extent = self._get_temporal_extent(dataset) + variables = self.get_variable_ids(dataset) + general_metadata = self._get_general_metadata(dataset) except ValueError as e: raise ValueError(f"Metadata extraction failed: {e}") From f6e01ccc893378a7fd9cb10813053027b268b8cf Mon Sep 17 00:00:00 2001 From: konstntokas Date: Wed, 12 Aug 2026 16:54:38 +0200 Subject: [PATCH 03/14] mini error found --- a | 1 - b | 1 - deep_code/tools/new.py | 42 ++++++++++++++++++++++++++++++-------- deep_code/tools/prr.py | 2 +- deep_code/tools/publish.py | 4 ++-- 5 files changed, 36 insertions(+), 14 deletions(-) delete mode 100644 a delete mode 100644 b diff --git a/a b/a deleted file mode 100644 index 9e26dfe..0000000 --- a/a +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/b b/b deleted file mode 100644 index 9e26dfe..0000000 --- a/b +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index ee1d651..a17f2a2 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -48,8 +48,14 @@ def generate_workflow_template(output_path: Optional[str] = None) -> str: with open(output_path, "w") as f: f.write("# Workflow Configuration Template\n") f.write("# Replace all [PLACEHOLDER] values with your actual data\n\n") - f.write(yaml.dump(workflow_template, sort_keys=False, width=1000, - default_flow_style=False)) + f.write( + yaml.dump( + workflow_template, + sort_keys=False, + width=1000, + default_flow_style=False, + ) + ) @staticmethod def generate_dataset_template(output_path: Optional[str] = None) -> str: @@ -69,7 +75,9 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: optional = { "osc_project_url": "[OPTIONAL: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection. Defaults to the existing DeepESDL project collection]", - "osc_themes": ["[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]"], + "osc_themes": [ + "[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]" + ], "osc_region": "[OPTIONAL: geographical coverage, e.g. Global]", "dataset_status": "[OPTIONAL: ongoing | completed | planned (default: ongoing)]", "description": "[OPTIONAL: human-readable description of the dataset. Overrides the description attribute in the Zarr store if set]", @@ -77,8 +85,10 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: "visualisation_link": "[OPTIONAL: URL to a visualisation of the dataset (e.g. xcube Viewer, WMS)]", "osc_project": "[OPTIONAL: OSC project ID (e.g. deep-earth-system-data-lab). Defaults to deep-earth-system-data-lab]", "osc_project_title": "[OPTIONAL: display title of the OSC project as it appears in the catalog (e.g. DeepESDL). Defaults to a formatted version of osc_project if omitted]", - "access_link": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public/{dataset_id}]", - "cf_parameter": [{"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"}], + "access_link_root": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public]", + "cf_parameter": [ + {"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"} + ], } # Fields used only by `deep-code generate-prr-collection` to build a @@ -90,7 +100,9 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: "osc_contract_number": "[PRR-REQUIRED: ESA contract identifier, e.g. 4000114410/15/NL/BW]", "osc_project_website": "[PRR-REQUIRED: project website URL — falls back to osc_project_url / documentation_link]", "osc_project_description": "[PRR-REQUIRED: multi-line project description — falls back to description]", - "osc_missions": ["[PRR-REQUIRED: satellite mission name(s), e.g. sentinel-3]"], + "osc_missions": [ + "[PRR-REQUIRED: satellite mission name(s), e.g. sentinel-3]" + ], "thumbnail": "[PRR-REQUIRED: URL to a collection thumbnail image (jpeg/png/webp)]", "thumbnail_media_type": "[OPTIONAL: thumbnail MIME type — guessed from the URL suffix if omitted]", "sci_doi": "[OPTIONAL: dataset DOI, e.g. 10.1000/xyz123 (not a DOI link)]", @@ -114,11 +126,23 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: f.write("# Dataset Configuration Template\n") f.write("# Replace all [PLACEHOLDER] values with your actual data\n\n") f.write("# --- REQUIRED fields ---\n") - f.write(yaml.dump(required, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + yaml.dump( + required, sort_keys=False, width=1000, default_flow_style=False + ) + ) f.write("\n# --- OPTIONAL fields ---\n") - f.write(yaml.dump(optional, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + yaml.dump( + optional, sort_keys=False, width=1000, default_flow_style=False + ) + ) f.write( "\n# --- PRR fields (for `deep-code generate-prr-collection`) ---\n" ) - f.write(yaml.dump(prr, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + yaml.dump( + prr, sort_keys=False, width=1000, default_flow_style=False + ) + ) f.write(stac_catalog_comment) diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index 5eff849..fff44aa 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -76,7 +76,7 @@ def generate_prr_collection( workflow_title=config.get("workflow_title") or "", license_type=license_type, documentation_link=config.get("documentation_link"), - access_link=config.get("access_link"), + access_link_root=config.get("access_link_root"), osc_status=config.get("dataset_status") or "ongoing", osc_region=config.get("osc_region") or "Global", osc_themes=config.get("osc_themes"), diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index 099e17d..c0db153 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -295,7 +295,7 @@ def publish_dataset( ) self.collection_id = self.dataset_config.get("collection_id") documentation_link = self.dataset_config.get("documentation_link") - access_link = self.dataset_config.get("access_link") + access_link_root = self.dataset_config.get("access_link_root") dataset_status = self.dataset_config.get("dataset_status") or "ongoing" osc_region = self.dataset_config.get("osc_region") osc_themes = self.dataset_config.get("osc_themes") @@ -333,7 +333,7 @@ def publish_dataset( workflow_title=self.workflow_title, license_type=license_type, documentation_link=documentation_link, - access_link=access_link, + access_link_root=access_link_root, osc_status=dataset_status, osc_region=osc_region, osc_themes=osc_themes, From 7e9f98da514701d8819c88e36c0d48864944d471 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 08:08:05 +0200 Subject: [PATCH 04/14] error in time extent --- .../utils/test_dataset_stac_generator.py | 35 ++++++++++++++++--- deep_code/utils/dataset_stac_generator.py | 7 ++-- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index a564ef0..b9a5bae 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -1107,10 +1107,37 @@ def test_build_prr_collection_no_themes(self, mock_open_ds): def test_save_prr_collection_writes_tree(self): with tempfile.TemporaryDirectory() as tmp: - with self.assertRaisesRegex(AttributeError, "tzinfo"): - self.gen.save_prr_collection(tmp) + out = self.gen.save_prr_collection(tmp) + self.assertEqual(out, tmp) + + collection_path = os.path.join(tmp, "collection.json") + item_path = os.path.join(tmp, "prr-collection", "prr-collection.json") + self.assertTrue(os.path.isfile(collection_path)) + self.assertTrue(os.path.isfile(item_path)) + + # Files are plain-JSON serialisable (no leftover Python objects). + with open(collection_path) as f: + coll_dict = json.load(f) + with open(item_path) as f: + item_dict = json.load(f) + + self.assertEqual(coll_dict["type"], "Collection") + self.assertEqual(item_dict["type"], "Feature") + + # Structural links are relative; the Item link points at the child. + item_link = next(lnk for lnk in coll_dict["links"] if lnk["rel"] == "item") + self.assertFalse(item_link["href"].startswith("s3://")) + self.assertTrue(item_link["href"].endswith(".json")) + + # Asset hrefs stay absolute (the data lives on S3). + self.assertEqual( + item_dict["assets"]["zarr-data"]["href"], "s3://bucket/test.zarr" + ) def test_save_prr_collection_readable_by_pystac(self): with tempfile.TemporaryDirectory() as tmp: - with self.assertRaisesRegex(AttributeError, "tzinfo"): - self.gen.save_prr_collection(tmp) + self.gen.save_prr_collection(tmp) + coll = Collection.from_file(os.path.join(tmp, "collection.json")) + items = list(coll.get_items()) + self.assertEqual(len(items), 1) + self.assertIn(DATACUBE_SCHEMA_URI, items[0].stac_extensions) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index d93d561..0a95fda 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -207,10 +207,13 @@ def _union_temporal_extent(items: list[Item]) -> TemporalExtent: """Merge multiple dataset temporal extents into a single interval.""" intervals = np.array( [ - [item.properties["start_datetime"], item.properties["end_datetime"]] + [ + datetime.fromisoformat(item.properties["start_datetime"]), + datetime.fromisoformat(item.properties["end_datetime"]), + ] for item in items ] - ).astype(np.datetime64) + ) return TemporalExtent([[min(intervals[:, 0]), max(intervals[:, 1])]]) def _get_general_metadata(self, dataset: xr.Dataset) -> dict: From 79d371496b9ae4792623932517ad5b5e4577b6ef Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 09:52:54 +0200 Subject: [PATCH 05/14] relative self href --- deep_code/tools/prr.py | 30 ++++++++--------------- deep_code/utils/dataset_stac_generator.py | 19 ++++++++------ 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index fff44aa..9824677 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -41,25 +41,6 @@ def generate_prr_collection( collection_id = config.get("collection_id") license_type = config.get("license_type") items_config_raw = config.get("items_config") - - if items_config_raw: - items_config = [ - ItemConfig( - dataset_id=item_config["dataset_id"], - item_id=item_config["item_id"], - ) - for item_config in items_config_raw - ] - else: - dataset_id = config.get("dataset_id") - item_id = config.get("item_id") or collection_id - if not dataset_id or not collection_id: - raise ValueError( - "At least one item configuration must be provided in the dataset config, " - "along with 'collection_id'." - ) - items_config = [ItemConfig(dataset_id=dataset_id, item_id=item_id)] - if not collection_id: raise ValueError("collection_id is required in the dataset config.") if not license_type: @@ -67,11 +48,20 @@ def generate_prr_collection( "license_type is required in the dataset config. " "Provide an SPDX identifier (e.g. 'CC-BY-4.0', 'MIT', 'proprietary')." ) + if not items_config_raw: + raise ValueError("items_config is required in the dataset config.") + items_config = [ + ItemConfig( + dataset_id=item_config["dataset_id"], + item_id=item_config["item_id"], + ) + for item_config in items_config_raw + ] logger.info(f"Generating PRR STAC collection for '{collection_id}'.") generator = OscDatasetStacGenerator( - items_config=items_config, collection_id=collection_id, + items_config=items_config, workflow_id=config.get("workflow_id") or "", workflow_title=config.get("workflow_title") or "", license_type=license_type, diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 0a95fda..c2d78f9 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -720,6 +720,7 @@ def build_zarr_stac_catalog_file_dict( {stac_catalog_s3_root}/ ├── catalog.json # STAC Catalog (root) └── {collection_id}/ + └── collection.json └── items/ ├── {item_id_0}.json # STAC Item (whole Zarr) └── {item_id_1}.json # STAC Item (whole Zarr) @@ -767,13 +768,13 @@ def build_zarr_stac_catalog_file_dict( ) return file_dict - # ------------------------------------------------------------------- # - # PRR (Project Results Repository) style output # - # # - # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # - # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # - # the plain catalog.json/items/*.json under ``{root}/prr/``. # - # ------------------------------------------------------------------- # + # --------------------------------------------------------------------- # + # PRR (Project Results Repository) style output # + # # + # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # + # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # + # the plain {collection_id}/items/{item_id}.json under ``{root}/prr/``. # + # --------------------------------------------------------------------- # @staticmethod def _get_epsg(dataset: xr.Dataset) -> int: @@ -945,6 +946,7 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: roles=["metadata"], ), ) + item.set_self_href(f"./{self.collection_id}/items/{item_config.item_id}.json") self.logger.info( f"PRR STAC Item '{item_config.item_id}' built for '{self.collection_id}'." ) @@ -974,6 +976,7 @@ def build_prr_collection(self) -> Collection: license=self.license_type, title=self.collection_id, ) + collection.set_self_href(f"./{self.collection_id}/collection.json") osc_extension = OscExtension.add_to(collection) osc_extension.osc_project = self.osc_project @@ -1115,8 +1118,8 @@ def save_prr_collection(self, output_dir: str) -> str: ready to inspect or submit to the ESA EarthCODE PRR endpoint:: {output_dir}/ - ├── collection.json # STAC Collection (root) └── {collection_id}/ + └── collection.json # STAC Collection (root) └── items └── {item_id_0}.json # datacube Item (whole Zarr) └── {item_id_1}.json # datacube Item (whole Zarr) From 604f57d81cadc84c6c07080c25470ee1f3b3a20b Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 10:01:26 +0200 Subject: [PATCH 06/14] remove normalize_href --- deep_code/utils/dataset_stac_generator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index c2d78f9..fb85a4e 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -958,8 +958,8 @@ def build_prr_collection(self) -> Collection: The Collection carries OSC extension fields (``osc:type``, ``osc:status``, ``osc:variables``, ``osc:missions``, ``themes``), a ``cf:parameter`` list and ``processing:datetime`` — aligning with the ESA EarthCODE PRR - endpoint (e.g. ``eoresults.esa.int``). The Item is added as a child so a - subsequent ``normalize_hrefs`` produces a self-contained tree. + endpoint (e.g. ``eoresults.esa.int``). The Item is added as a child, so + it produces a self-contained tree. """ items = [ self.build_prr_stac_item(item_config) for item_config in self.items_config @@ -1138,7 +1138,6 @@ def save_prr_collection(self, output_dir: str) -> str: f"'{output_dir}'." ) collection = self.build_prr_collection() - collection.normalize_hrefs(output_dir) collection.save(catalog_type=CatalogType.SELF_CONTAINED) self.logger.info(f"PRR STAC collection written to '{output_dir}'.") return output_dir From 904d37846086358bbabd1f9b154e7d7aa59064d6 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 10:17:37 +0200 Subject: [PATCH 07/14] add absolute hrefs --- deep_code/utils/dataset_stac_generator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index fb85a4e..68e2912 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -946,7 +946,7 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: roles=["metadata"], ), ) - item.set_self_href(f"./{self.collection_id}/items/{item_config.item_id}.json") + self.logger.info( f"PRR STAC Item '{item_config.item_id}' built for '{self.collection_id}'." ) @@ -976,7 +976,6 @@ def build_prr_collection(self) -> Collection: license=self.license_type, title=self.collection_id, ) - collection.set_self_href(f"./{self.collection_id}/collection.json") osc_extension = OscExtension.add_to(collection) osc_extension.osc_project = self.osc_project @@ -1138,6 +1137,15 @@ def save_prr_collection(self, output_dir: str) -> str: f"'{output_dir}'." ) collection = self.build_prr_collection() + + # Set absolute HREFs for writing. + collection_dir = f"{output_dir}/{self.collection_id}" + items_dir = f"{collection_dir}/items" + collection.set_self_href(f"{collection_dir}/collection.json") + for item in collection.get_items(): + item.set_self_href(f"{items_dir}/{item.id}.json") + + # Write the collection and its children. collection.save(catalog_type=CatalogType.SELF_CONTAINED) self.logger.info(f"PRR STAC collection written to '{output_dir}'.") return output_dir From 8de42242bf87f1949441ae70192e2b604ad2a355 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 11:31:57 +0200 Subject: [PATCH 08/14] link changed --- a | 1 + b | 1 + deep_code/tools/prr.py | 3 +- deep_code/utils/dataset_stac_generator.py | 108 ++++++++++++++++------ 4 files changed, 84 insertions(+), 29 deletions(-) create mode 100644 a create mode 100644 b diff --git a/a b/a new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/a @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/b b/b new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/b @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index 9824677..a264e54 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -74,7 +74,8 @@ def generate_prr_collection( cf_params=config.get("cf_parameter"), visualisation_link=config.get("visualisation_link"), description=config.get("description"), - **({"osc_project": config["osc_project"]} if config.get("osc_project") else {}), + osc_project=config.get("osc_project"), + osc_project_title=config.get("osc_project_title"), osc_project_url=config.get("osc_project_url"), # PRR-specific project metadata. osc_initiative=config.get("osc_initiative") or "earthcode", diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 68e2912..1ead07e 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -11,6 +11,7 @@ import numpy as np import pandas as pd +import pyproj from pystac import ( Asset, Catalog, @@ -123,42 +124,36 @@ def __init__( self.sci_citation = sci_citation self.logger = logging.getLogger(__name__) - @staticmethod - def _get_spatial_extent(dataset: xr.Dataset) -> SpatialExtent: - """Extract spatial extent from the dataset.""" + def _get_spatial_extent(self, dataset: xr.Dataset) -> SpatialExtent: + """Extract the spatial extent and return it in EPSG:4326.""" + + epsg = self._get_epsg(dataset) + if {"lon", "lat"}.issubset(dataset.coords): - # For regular gridding - lon_min, lon_max = ( - float(dataset.lon.min()), - float(dataset.lon.max()), - ) - lat_min, lat_max = ( - float(dataset.lat.min()), - float(dataset.lat.max()), - ) - return SpatialExtent([[lon_min, lat_min, lon_max, lat_max]]) + x_name, y_name = "lon", "lat" elif {"longitude", "latitude"}.issubset(dataset.coords): - # For regular gridding with 'longitude' and 'latitude' - lon_min, lon_max = ( - float(dataset.longitude.min()), - float(dataset.longitude.max()), - ) - lat_min, lat_max = ( - float(dataset.latitude.min()), - float(dataset.latitude.max()), - ) - return SpatialExtent([[lon_min, lat_min, lon_max, lat_max]]) + x_name, y_name = "longitude", "latitude" elif {"x", "y"}.issubset(dataset.coords): - # For irregular gridding - x_min, x_max = (float(dataset.x.min()), float(dataset.x.max())) - y_min, y_max = (float(dataset.y.min()), float(dataset.y.max())) - return SpatialExtent([[x_min, y_min, x_max, y_max]]) + x_name, y_name = "x", "y" else: raise ValueError( "Dataset does not have recognized spatial coordinates " - "('lon', 'lat' or 'x', 'y')." + "('lon', 'lat'), ('longitude', 'latitude'), or ('x', 'y')." ) + x_min = float(dataset[x_name].min()) + x_max = float(dataset[x_name].max()) + y_min = float(dataset[y_name].min()) + y_max = float(dataset[y_name].max()) + + epsg = self._get_epsg(dataset) + + if epsg != 4326: + transformer = pyproj.Transformer.from_crs(epsg, 4326, always_xy=True) + bbox = transformer.transform_bounds(x_min, y_min, x_max, y_max) + + return SpatialExtent([bbox]) + @staticmethod def _get_temporal_extent(dataset: xr.Dataset) -> TemporalExtent: """Extract temporal extent from the dataset.""" @@ -924,6 +919,34 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: "cube:variables": self._get_cube_variables(dataset), }, ) + item.add_link( + Link( + rel="collection", + target="../collection.json", + media_type="application/json", + ) + ) + item.add_link( + Link( + rel="parent", + target="../collection.json", + media_type="application/json", + ) + ) + item.add_link( + Link( + rel="root", + target="../../", + media_type="application/json", + ) + ) + item.add_link( + Link( + rel="self", + target=f"./{item_config.item_id}.json", + media_type="application/geo+json", + ) + ) item.stac_extensions.append(DATACUBE_SCHEMA_URI) # Asset hrefs stay absolute (the Zarr lives on S3); only the structural # links become relative when the tree is normalised locally. @@ -976,6 +999,35 @@ def build_prr_collection(self) -> Collection: license=self.license_type, title=self.collection_id, ) + collection.stac_version = "1.0.0" + collection.add_link( + Link( + rel="parent", + target="../", + media_type="application/json", + ) + ) + collection.add_link( + Link( + rel="root", + target="../", + media_type="application/json", + ) + ) + collection.add_link( + Link( + rel="self", + target="./collection.json", + media_type="application/json", + ) + ) + collection.add_link( + Link( + rel="items", + target="./items", + media_type="application/geo+json", + ) + ) osc_extension = OscExtension.add_to(collection) osc_extension.osc_project = self.osc_project From 9352ade660bafe8b840e15ab20e30772eb107f0d Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 11:45:40 +0200 Subject: [PATCH 09/14] links missiong --- .../utils/test_dataset_stac_generator.py | 10 ++- deep_code/utils/dataset_stac_generator.py | 62 ++----------------- 2 files changed, 11 insertions(+), 61 deletions(-) diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index b9a5bae..04a5a00 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -1110,8 +1110,10 @@ def test_save_prr_collection_writes_tree(self): out = self.gen.save_prr_collection(tmp) self.assertEqual(out, tmp) - collection_path = os.path.join(tmp, "collection.json") - item_path = os.path.join(tmp, "prr-collection", "prr-collection.json") + collection_path = os.path.join(tmp, "prr-collection", "collection.json") + item_path = os.path.join( + tmp, "prr-collection", "items", "prr-collection.json" + ) self.assertTrue(os.path.isfile(collection_path)) self.assertTrue(os.path.isfile(item_path)) @@ -1137,7 +1139,9 @@ def test_save_prr_collection_writes_tree(self): def test_save_prr_collection_readable_by_pystac(self): with tempfile.TemporaryDirectory() as tmp: self.gen.save_prr_collection(tmp) - coll = Collection.from_file(os.path.join(tmp, "collection.json")) + coll = Collection.from_file( + os.path.join(tmp, "prr-collection", "collection.json") + ) items = list(coll.get_items()) self.assertEqual(len(items), 1) self.assertIn(DATACUBE_SCHEMA_URI, items[0].stac_extensions) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 1ead07e..b0f070e 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -150,9 +150,11 @@ def _get_spatial_extent(self, dataset: xr.Dataset) -> SpatialExtent: if epsg != 4326: transformer = pyproj.Transformer.from_crs(epsg, 4326, always_xy=True) - bbox = transformer.transform_bounds(x_min, y_min, x_max, y_max) + x_min, y_min, x_max, y_max = transformer.transform_bounds( + x_min, y_min, x_max, y_max + ) - return SpatialExtent([bbox]) + return SpatialExtent([[x_min, y_min, x_max, y_max]]) @staticmethod def _get_temporal_extent(dataset: xr.Dataset) -> TemporalExtent: @@ -919,34 +921,6 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: "cube:variables": self._get_cube_variables(dataset), }, ) - item.add_link( - Link( - rel="collection", - target="../collection.json", - media_type="application/json", - ) - ) - item.add_link( - Link( - rel="parent", - target="../collection.json", - media_type="application/json", - ) - ) - item.add_link( - Link( - rel="root", - target="../../", - media_type="application/json", - ) - ) - item.add_link( - Link( - rel="self", - target=f"./{item_config.item_id}.json", - media_type="application/geo+json", - ) - ) item.stac_extensions.append(DATACUBE_SCHEMA_URI) # Asset hrefs stay absolute (the Zarr lives on S3); only the structural # links become relative when the tree is normalised locally. @@ -1000,34 +974,6 @@ def build_prr_collection(self) -> Collection: title=self.collection_id, ) collection.stac_version = "1.0.0" - collection.add_link( - Link( - rel="parent", - target="../", - media_type="application/json", - ) - ) - collection.add_link( - Link( - rel="root", - target="../", - media_type="application/json", - ) - ) - collection.add_link( - Link( - rel="self", - target="./collection.json", - media_type="application/json", - ) - ) - collection.add_link( - Link( - rel="items", - target="./items", - media_type="application/geo+json", - ) - ) osc_extension = OscExtension.add_to(collection) osc_extension.osc_project = self.osc_project From a1cf6cf963e09f10778a0038d894e8808adb6419 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 12:17:16 +0200 Subject: [PATCH 10/14] allow cf spatial ref --- .../utils/test_dataset_stac_generator.py | 6 ++-- deep_code/utils/dataset_stac_generator.py | 33 ++++++++++--------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 04a5a00..097ed4a 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -907,8 +907,8 @@ def setUp(self): # ---- helpers ---- - def test_get_epsg_from_spatial_ref(self): - self.assertEqual(self.gen._get_epsg(self.dataset), 3035) + def test_get_crs_from_spatial_ref(self): + self.assertEqual(self.gen._get_crs(self.dataset).to_epsg(), 3035) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_get_epsg_default_4326(self, mock_open_ds): @@ -930,7 +930,7 @@ def test_get_epsg_default_4326(self, mock_open_ds): workflow_title="WF", license_type="CC-BY-4.0", ) - self.assertEqual(gen._get_epsg(ds), 4326) + self.assertEqual(gen._get_crs(ds).to_epsg(), 4326) def test_get_cube_dimensions(self): dims = self.gen._get_cube_dimensions(self.dataset) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index b0f070e..558371f 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -127,8 +127,6 @@ def __init__( def _get_spatial_extent(self, dataset: xr.Dataset) -> SpatialExtent: """Extract the spatial extent and return it in EPSG:4326.""" - epsg = self._get_epsg(dataset) - if {"lon", "lat"}.issubset(dataset.coords): x_name, y_name = "lon", "lat" elif {"longitude", "latitude"}.issubset(dataset.coords): @@ -146,10 +144,10 @@ def _get_spatial_extent(self, dataset: xr.Dataset) -> SpatialExtent: y_min = float(dataset[y_name].min()) y_max = float(dataset[y_name].max()) - epsg = self._get_epsg(dataset) + crs = self._get_crs(dataset) - if epsg != 4326: - transformer = pyproj.Transformer.from_crs(epsg, 4326, always_xy=True) + if crs.to_epsg() != 4326: + transformer = pyproj.Transformer.from_crs(crs, 4326, always_xy=True) x_min, y_min, x_max, y_max = transformer.transform_bounds( x_min, y_min, x_max, y_max ) @@ -774,7 +772,7 @@ def build_zarr_stac_catalog_file_dict( # --------------------------------------------------------------------- # @staticmethod - def _get_epsg(dataset: xr.Dataset) -> int: + def _get_crs(dataset: xr.Dataset) -> pyproj.CRS: """Best-effort EPSG code for the dataset, defaulting to 4326. Reads an ``spatial_epsg``/``epsg`` attribute from a ``crs`` or @@ -784,13 +782,16 @@ def _get_epsg(dataset: xr.Dataset) -> int: for var_name in ("spatial_ref", "crs"): if var_name in dataset.variables: attrs = dataset[var_name].attrs - for key in ("spatial_epsg", "epsg", "EPSG"): - if key in attrs: - try: - return int(attrs[key]) - except (TypeError, ValueError): - pass - return 4326 + try: + return pyproj.CRS.from_cf(attrs) + except pyproj.exceptions.CRSError: + for key in ("spatial_epsg", "epsg", "EPSG"): + if key in attrs: + try: + return pyproj.CRS.from_epsg(attrs[key]) + except (TypeError, ValueError): + pass + return pyproj.CRS.from_epsg(4326) def _get_cube_dimensions(self, dataset: xr.Dataset) -> dict[str, dict]: """Build the ``cube:dimensions`` object from the dataset coordinates. @@ -800,7 +801,7 @@ def _get_cube_dimensions(self, dataset: xr.Dataset) -> dict[str, dict]: dimension, and any remaining index coordinate is emitted as an additional dimension. """ - epsg = self._get_epsg(dataset) + crs = self._get_crs(dataset) x_names = {"lon", "longitude", "x"} y_names = {"lat", "latitude", "y"} dimensions: dict[str, dict] = {} @@ -815,14 +816,14 @@ def _get_cube_dimensions(self, dataset: xr.Dataset) -> dict[str, dict]: "type": "spatial", "axis": "x", "extent": [float(coord.min()), float(coord.max())], - "reference_system": epsg, + "reference_system": crs.to_epsg(), } elif lname in y_names: dimensions[name] = { "type": "spatial", "axis": "y", "extent": [float(coord.min()), float(coord.max())], - "reference_system": epsg, + "reference_system": crs.to_epsg(), } elif lname == "time": time_min = pd.to_datetime(coord.min().values).to_pydatetime() From 334eca55dcdf053bb2966509ab48cf296c2cea80 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 14:50:05 +0200 Subject: [PATCH 11/14] ready for review --- a | 1 - b | 1 - deep_code/tests/tools/test_publish.py | 86 +++++++++++++------ .../utils/test_dataset_stac_generator.py | 8 +- deep_code/tools/new.py | 4 +- deep_code/tools/prr.py | 6 ++ deep_code/tools/publish.py | 58 ++++++------- deep_code/utils/dataset_stac_generator.py | 42 ++++----- deep_code/utils/ogc_api_record.py | 4 +- 9 files changed, 118 insertions(+), 92 deletions(-) delete mode 100644 a delete mode 100644 b diff --git a/a b/a deleted file mode 100644 index 9e26dfe..0000000 --- a/a +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/b b/b deleted file mode 100644 index 9e26dfe..0000000 --- a/b +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/deep_code/tests/tools/test_publish.py b/deep_code/tests/tools/test_publish.py index 4df212e..9a1cd0f 100644 --- a/deep_code/tests/tools/test_publish.py +++ b/deep_code/tests/tools/test_publish.py @@ -11,6 +11,7 @@ from deep_code.tools.publish import Publisher from deep_code.utils.ogc_api_record import LinksBuilder +from deep_code.utils.dataset_stac_generator import ItemConfig class TestPublisher(unittest.TestCase): @@ -24,7 +25,7 @@ def setUp(self, mock_github_publisher, mock_fsspec_open): # Mock dataset and workflow config files self.dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], } self.workflow_config = { "properties": {"title": "Test Workflow"}, @@ -94,7 +95,7 @@ def test_read_config_files(self): # Mock dataset and workflow config files dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], } workflow_config = { "properties": {"title": "Test Workflow"}, @@ -138,7 +139,7 @@ def test_publish_mode_routing(self, mock_wf, mock_ds, mock_s3): self.publisher.dataset_config = { "stac_catalog_s3_root": "s3://bucket/stac/", "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], } self.publisher.gh_publisher.publish_files.return_value = "PR_URL" @@ -198,7 +199,6 @@ def test_publish_builds_pr_params(self, mock_wf, mock_ds, mock_s3): assert "dataset: col" in kwargs["pr_title"] assert "workflow/experiment: wf" in kwargs["pr_title"] - # ------------------------------------------------------------------ # S3 credential resolution # ------------------------------------------------------------------ @@ -262,9 +262,7 @@ def test_write_stac_catalog_to_s3(self, mock_fsspec_open): "s3://bucket/catalog.json": {"type": "Catalog", "id": "test"}, "s3://bucket/col/item.json": {"type": "Feature", "id": "item"}, } - self.publisher._write_stac_catalog_to_s3( - file_dict, {"key": "k", "secret": "s"} - ) + self.publisher._write_stac_catalog_to_s3(file_dict, {"key": "k", "secret": "s"}) self.assertEqual(mock_fsspec_open.call_count, 2) mock_fsspec_open.assert_any_call( @@ -283,9 +281,7 @@ def test_write_stac_catalog_to_s3(self, mock_fsspec_open): def test_publish_writes_zarr_stac_to_s3_when_configured( self, mock_publish_ds, mock_fsspec_open ): - self.publisher.dataset_config["stac_catalog_s3_root"] = ( - "s3://test-bucket/stac/" - ) + self.publisher.dataset_config["stac_catalog_s3_root"] = "s3://test-bucket/stac/" mock_ctx = MagicMock() mock_ctx.__enter__ = MagicMock(return_value=MagicMock()) @@ -332,8 +328,10 @@ def test_publish_dataset_creates_project_collection_when_missing( MockGenerator.return_value = mock_gen self.publisher.dataset_config = { - "dataset_id": "test-dataset", "collection_id": "test-collection", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", "license_type": "CC-BY-4.0", "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", } @@ -342,9 +340,14 @@ def test_publish_dataset_creates_project_collection_when_missing( # Project collection is missing; all other file_exists calls return True self.publisher.gh_publisher.github_automation.file_exists.return_value = False - with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \ - patch.object(self.publisher, "_update_variable_catalogs"): - file_dict = self.publisher.publish_dataset(write_to_file=False) + with patch("deep_code.tools.publish.open_dataset", return_value=object()): + with ( + patch.object( + self.publisher, "_update_and_add_to_file_dict" + ) as mock_update, + patch.object(self.publisher, "_update_variable_catalogs"), + ): + file_dict = self.publisher.publish_dataset(write_to_file=False) mock_gen.build_project_collection.assert_called_once() self.assertIn("projects/test-project/collection.json", file_dict) @@ -367,8 +370,10 @@ def test_publish_dataset_updates_project_collection_when_exists( MockGenerator.return_value = mock_gen self.publisher.dataset_config = { - "dataset_id": "test-dataset", "collection_id": "test-collection", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", "license_type": "CC-BY-4.0", "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", } @@ -377,9 +382,14 @@ def test_publish_dataset_updates_project_collection_when_exists( # Project collection already exists self.publisher.gh_publisher.github_automation.file_exists.return_value = True - with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \ - patch.object(self.publisher, "_update_variable_catalogs"): - self.publisher.publish_dataset(write_to_file=False) + with patch("deep_code.tools.publish.open_dataset", return_value=object()): + with ( + patch.object( + self.publisher, "_update_and_add_to_file_dict" + ) as mock_update, + patch.object(self.publisher, "_update_variable_catalogs"), + ): + self.publisher.publish_dataset(write_to_file=False) mock_gen.build_project_collection.assert_not_called() @@ -390,7 +400,9 @@ def test_publish_dataset_updates_project_collection_when_exists( def test_publish_dataset_raises_when_stac_root_missing(self): self.publisher.dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", "license_type": "CC-BY-4.0", } with pytest.raises(ValueError, match="stac_catalog_s3_root"): @@ -402,14 +414,24 @@ def test_publish_dataset_raises_when_no_dataset_config(self): self.publisher.publish_dataset(write_to_file=False) def test_publish_dataset_raises_when_ids_missing(self): - self.publisher.dataset_config = {"collection_id": "", "dataset_id": ""} - with pytest.raises(ValueError, match="item configuration"): + self.publisher.dataset_config = { + "collection_id": "", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", + "license_type": "CC-BY-4.0", + "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", + } + with pytest.raises(ValueError, match="collection_id missing"): self.publisher.publish_dataset(write_to_file=False) def test_publish_dataset_raises_when_license_missing(self): self.publisher.dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", + "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", } with pytest.raises(ValueError, match="license_type is required"): self.publisher.publish_dataset(write_to_file=False) @@ -433,14 +455,18 @@ def test_update_and_add_to_file_dict(self): file_dict = {} self.publisher.gh_publisher.github_automation.local_clone_dir = "/tmp" update_method = MagicMock(return_value={"key": "value"}) - self.publisher._update_and_add_to_file_dict(file_dict, "some/catalog.json", update_method) + self.publisher._update_and_add_to_file_dict( + file_dict, "some/catalog.json", update_method + ) update_method.assert_called_once() assert any("some/catalog.json" in str(k) for k in file_dict) def test_update_variable_catalogs_creates_new_when_missing(self): mock_gen = MagicMock() mock_gen.variables_metadata = {"var1": {"variable_id": "var1"}} - mock_gen.build_variable_catalog.return_value.to_dict.return_value = {"id": "var1"} + mock_gen.build_variable_catalog.return_value.to_dict.return_value = { + "id": "var1" + } self.publisher.gh_publisher.github_automation.file_exists.return_value = False file_dict = {} @@ -509,8 +535,12 @@ def test_generate_workflow_records_mode_workflow(self, MockRG, MockLinks, MockWF @patch("deep_code.tools.publish.WorkflowAsOgcRecord") @patch("deep_code.tools.publish.LinksBuilder") @patch("deep_code.tools.publish.OSCWorkflowOGCApiRecordGenerator") - def test_generate_workflow_records_mode_all(self, MockRG, MockLinks, MockWF, MockExp): - mock_rg, mock_props, mock_wf_record, mock_exp_record = self._setup_workflow_mocks() + def test_generate_workflow_records_mode_all( + self, MockRG, MockLinks, MockWF, MockExp + ): + mock_rg, mock_props, mock_wf_record, mock_exp_record = ( + self._setup_workflow_mocks() + ) MockRG.return_value = mock_rg MockWF.return_value = mock_wf_record MockExp.return_value = mock_exp_record @@ -585,8 +615,8 @@ class TestParseGithubNotebookUrl: ], ) def test_valid_urls(self, url, repo_url, repo_name, branch, file_path): - got_repo_url, got_repo_name, got_branch, got_file_path = LinksBuilder._parse_github_notebook_url( - url + got_repo_url, got_repo_name, got_branch, got_file_path = ( + LinksBuilder._parse_github_notebook_url(url) ) assert got_repo_url == repo_url assert got_repo_name == repo_name diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 097ed4a..9c9cd42 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -472,7 +472,7 @@ def test_build_zarr_stac_item_structure(self): # Self href self.assertEqual( item.self_href, - "s3://test-bucket/stac/my-collection/mock-collection-id/items/mock-collection-id.json", + "s3://test-bucket/stac/my-collection/mock-collection-id/item.json", ) # Required link rels @@ -538,7 +538,7 @@ def test_build_zarr_stac_catalog_file_dict_content(self): @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_build_zarr_stac_catalog_file_dict_multiple_items(self, mock_open_ds): - """Multiple item configurations produce multiple item files and links.""" + """Only the first item configuration is emitted in the STAC file dict.""" mock_open_ds.return_value = self.mock_dataset gen = OscDatasetStacGenerator( collection_id="multi-collection", @@ -560,14 +560,14 @@ def test_build_zarr_stac_catalog_file_dict_multiple_items(self, mock_open_ds): "multi-collection/items/first-item.json", file_dict, ) - self.assertIn( + self.assertNotIn( "s3://test-bucket/stac/multi-collection/" "multi-collection/items/second-item.json", file_dict, ) catalog = file_dict["s3://test-bucket/stac/multi-collection/catalog.json"] item_links = [lnk for lnk in catalog["links"] if lnk["rel"] == "item"] - self.assertEqual(len(item_links), 2) + self.assertEqual(len(item_links), 1) def test_build_dataset_stac_collection_adds_s3_catalog_via_link(self): """A 'via' link (STAC browser) and a 'child' link (HTTPS catalog) are added diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index a17f2a2..7264fb6 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -71,10 +71,11 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: "item_id": "[REQUIRED: unique STAC item id, no spaces — use hyphens]", } ], + "osc_project": "[OPTIONAL: OSC project ID (e.g. deep-earth-system-data-lab). Defaults to deep-earth-system-data-lab]", + "osc_project_url": "[OPTIONAL: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection.]", } optional = { - "osc_project_url": "[OPTIONAL: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection. Defaults to the existing DeepESDL project collection]", "osc_themes": [ "[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]" ], @@ -83,7 +84,6 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: "description": "[OPTIONAL: human-readable description of the dataset. Overrides the description attribute in the Zarr store if set]", "documentation_link": "[OPTIONAL: link to documentation, publication, or handbook]", "visualisation_link": "[OPTIONAL: URL to a visualisation of the dataset (e.g. xcube Viewer, WMS)]", - "osc_project": "[OPTIONAL: OSC project ID (e.g. deep-earth-system-data-lab). Defaults to deep-earth-system-data-lab]", "osc_project_title": "[OPTIONAL: display title of the OSC project as it appears in the catalog (e.g. DeepESDL). Defaults to a formatted version of osc_project if omitted]", "access_link_root": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public]", "cf_parameter": [ diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index a264e54..93b9e45 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -39,10 +39,16 @@ def generate_prr_collection( config = yaml.safe_load(file) or {} collection_id = config.get("collection_id") + osc_project = config.get("osc_project") + osc_project_url = config.get("osc_project_url") license_type = config.get("license_type") items_config_raw = config.get("items_config") if not collection_id: raise ValueError("collection_id is required in the dataset config.") + if not osc_project: + raise ValueError("osc_project is required in the dataset config.") + if not osc_project_url: + raise ValueError("osc_project_url is required in the dataset config.") if not license_type: raise ValueError( "license_type is required in the dataset config. " diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index c0db153..399f102 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -22,7 +22,11 @@ OSC_REPO_OWNER, WORKFLOW_BASE_CATALOG_SELF_HREF, ) -from deep_code.utils.dataset_stac_generator import ItemConfig, OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ( + ItemConfig, + OscDatasetStacGenerator, + open_dataset, +) from deep_code.utils.github_automation import GitHubAutomation from deep_code.utils.ogc_api_record import ( ExperimentAsOgcRecord, @@ -160,6 +164,8 @@ def __init__( # Values that may be set from configs self.collection_id: str | None = None + self.osc_project: str | None = None + self.osc_project_url: str | None = None self.workflow_title: str | None = None self.workflow_id: str | None = None @@ -250,30 +256,17 @@ def _update_variable_catalogs(self, generator, file_dict, variable_ids): @staticmethod def _build_items_config(dataset_config: dict[str, Any]) -> list[ItemConfig]: - """Build item configs from the dataset config. - - Supports the new ``items_config`` list while keeping the legacy - single-item ``dataset_id`` / ``item_id`` fields for backwards - compatibility. - """ + """Build item configs from the dataset config.""" items_config_raw = dataset_config.get("items_config") - if items_config_raw: - items_config = [ - ItemConfig( - dataset_id=item_config["dataset_id"], - item_id=item_config["item_id"], - ) - for item_config in items_config_raw - ] - else: - dataset_id = dataset_config.get("dataset_id") - collection_id = dataset_config.get("collection_id") - item_id = dataset_config.get("item_id") or collection_id - if not dataset_id: - raise ValueError( - "At least one item configuration must be provided in the dataset config." - ) - items_config = [ItemConfig(dataset_id=dataset_id, item_id=item_id)] + if not items_config_raw: + raise ValueError("items_config is required in the dataset config.") + items_config = [ + ItemConfig( + dataset_id=item_config["dataset_id"], + item_id=item_config["item_id"], + ) + for item_config in items_config_raw + ] return items_config def publish_dataset( @@ -302,13 +295,17 @@ def publish_dataset( cf_params = self.dataset_config.get("cf_parameter") license_type = self.dataset_config.get("license_type") visualisation_link = self.dataset_config.get("visualisation_link") - osc_project = self.dataset_config.get("osc_project") + self.osc_project = self.dataset_config.get("osc_project") osc_project_title = self.dataset_config.get("osc_project_title") - osc_project_url = self.dataset_config.get("osc_project_url") + self.osc_project_url = self.dataset_config.get("osc_project_url") description = self.dataset_config.get("description") if not self.collection_id: - raise ValueError("Collection ID missing in the config.") + raise ValueError("collection_id missing in the config.") + if not self.osc_project: + raise ValueError("osc_project missing in the config.") + if not self.osc_project_url: + raise ValueError("osc_project missing in the config.") if not license_type: raise ValueError( @@ -339,15 +336,16 @@ def publish_dataset( osc_themes=osc_themes, cf_params=cf_params, visualisation_link=visualisation_link, - **({"osc_project": osc_project} if osc_project else {}), + osc_project=self.osc_project, osc_project_title=osc_project_title, - osc_project_url=osc_project_url, + osc_project_url=self.osc_project_url, description=description, ) # Store so publish() can reuse it for zarr STAC catalog generation self._last_generator = generator - variable_ids = generator.get_variable_ids() + dataset = open_dataset(generator.items_config[0].dataset_id) + variable_ids = generator.get_variable_ids(dataset) ds_collection = generator.build_dataset_stac_collection( mode=mode, stac_catalog_s3_root=stac_catalog_s3_root ) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 558371f..a2c7a4c 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -646,7 +646,7 @@ def build_zarr_stac_item( now_iso = datetime.now(timezone.utc).isoformat() root = stac_catalog_s3_root.rstrip("/") catalog_href = f"{root}/catalog.json" - item_href = f"{root}/{self.collection_id}/items/{item_config.item_id}.json" + item_href = f"{root}/{self.collection_id}/item.json" osc_collection_href = ( "https://esa-earthcode.github.io/open-science-catalog-metadata" f"/products/{self.collection_id}/collection.json" @@ -715,10 +715,7 @@ def build_zarr_stac_catalog_file_dict( {stac_catalog_s3_root}/ ├── catalog.json # STAC Catalog (root) └── {collection_id}/ - └── collection.json - └── items/ - ├── {item_id_0}.json # STAC Item (whole Zarr) - └── {item_id_1}.json # STAC Item (whole Zarr) + └── item.json # STAC Item (whole Zarr) Args: stac_catalog_s3_root: S3 root URL (e.g. ``s3://my-bucket/stac/``). @@ -730,9 +727,9 @@ def build_zarr_stac_catalog_file_dict( f"Building STAC Catalog file dict for collection '{self.collection_id}' " f"at root '{stac_catalog_s3_root}'." ) + root = stac_catalog_s3_root.rstrip("/") catalog_href = f"{root}/catalog.json" - catalog = Catalog( id=f"{self.collection_id}-stac-catalog", description=f"STAC Catalog for {self.collection_id}", @@ -741,27 +738,24 @@ def build_zarr_stac_catalog_file_dict( catalog.add_link( Link(rel="root", target=catalog_href, media_type="application/json") ) - file_dict = {catalog_href: catalog.to_dict(transform_hrefs=False)} - for item_config in self.items_config: - item = self.build_zarr_stac_item(item_config, stac_catalog_s3_root) - item_href = f"{root}/{self.collection_id}/items/{item_config.item_id}.json" - catalog.add_link( - Link( - rel="item", - target=f"./{self.collection_id}/items/{item_config.item_id}.json", - media_type="application/json", - title=item_config.item_id, - ) + item_config = self.items_config[0] + item = self.build_zarr_stac_item(item_config, stac_catalog_s3_root) + catalog.add_link( + Link( + rel="item", + target=f"./{self.collection_id}/items/{item_config.item_id}.json", + media_type="application/json", + title=item_config.item_id, ) - file_dict[item_href] = item.to_dict(transform_hrefs=False) - - file_dict[catalog_href] = catalog.to_dict(transform_hrefs=False) - self.logger.info( - "STAC Catalog file dict ready: %s", - ", ".join(file_dict.keys()), ) - return file_dict + item_href = f"{root}/{self.collection_id}/items/{item_config.item_id}.json" + + self.logger.info(f"STAC Catalog file dict ready: {catalog_href}, {item_href}") + return { + catalog_href: catalog.to_dict(transform_hrefs=False), + item_href: item.to_dict(transform_hrefs=False), + } # --------------------------------------------------------------------- # # PRR (Project Results Repository) style output # diff --git a/deep_code/utils/ogc_api_record.py b/deep_code/utils/ogc_api_record.py index 4a46fb9..9b5c97c 100644 --- a/deep_code/utils/ogc_api_record.py +++ b/deep_code/utils/ogc_api_record.py @@ -110,7 +110,7 @@ def to_dict(self, value_name: str | None = None) -> dict[str, JsonValue]: class LinksBuilder: - def __init__(self, themes: list[str], jupyter_kernel_info: dict[str]): + def __init__(self, themes: list[str], jupyter_kernel_info: dict[str, Any]): self.themes = themes self.jupyter_kernel_info = jupyter_kernel_info self.theme_links = [] @@ -233,7 +233,7 @@ def make_related_link_for_opening_jnb_from_github( jupyter_notebook_url: str, title: str = "Open notebook on the DeepESDL platform", branch_override: str | None = None, - ) -> dict[str, str]: + ) -> list[dict[str, str]]: return [ { "rel": "related", From 9528fcd82aeaf540ba9f5bd11e97dd117de00aa4 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 15:18:57 +0200 Subject: [PATCH 12/14] ruff --- deep_code/cli/generate_config.py | 4 +- deep_code/cli/main.py | 1 - deep_code/tests/tools/test_publish.py | 1 - .../utils/test_dataset_stac_generator.py | 6 +-- .../tests/utils/test_github_automation.py | 13 +++-- deep_code/tools/new.py | 5 +- deep_code/tools/prr.py | 2 +- deep_code/tools/test.py | 2 +- deep_code/utils/dataset_stac_generator.py | 8 +-- deep_code/utils/helper.py | 18 ++++--- deep_code/utils/ogc_api_record.py | 12 ++--- deep_code/utils/ogc_record_generator.py | 18 +++---- docs/cli.md | 14 ++--- docs/configuration.md | 54 ++++++++++--------- docs/examples.md | 2 +- docs/python-api.md | 2 +- 16 files changed, 83 insertions(+), 79 deletions(-) diff --git a/deep_code/cli/generate_config.py b/deep_code/cli/generate_config.py index ce1c286..ba63d0d 100644 --- a/deep_code/cli/generate_config.py +++ b/deep_code/cli/generate_config.py @@ -18,5 +18,5 @@ help="Output directory for templates", ) def generate_config(output_dir): - TemplateGenerator.generate_workflow_template(f"{output_dir}/workflow_config.yaml") - TemplateGenerator.generate_dataset_template(f"{output_dir}/dataset_config.yaml") + TemplateGenerator.generate_workflow_template(f"{output_dir}/workflow.yaml") + TemplateGenerator.generate_dataset_template(f"{output_dir}/dataset.yaml") diff --git a/deep_code/cli/main.py b/deep_code/cli/main.py index ac306f5..dfc8295 100644 --- a/deep_code/cli/main.py +++ b/deep_code/cli/main.py @@ -15,7 +15,6 @@ @click.group() def main(): """Deep Code CLI.""" - pass main.add_command(publish) diff --git a/deep_code/tests/tools/test_publish.py b/deep_code/tests/tools/test_publish.py index 9a1cd0f..a411779 100644 --- a/deep_code/tests/tools/test_publish.py +++ b/deep_code/tests/tools/test_publish.py @@ -11,7 +11,6 @@ from deep_code.tools.publish import Publisher from deep_code.utils.ogc_api_record import LinksBuilder -from deep_code.utils.dataset_stac_generator import ItemConfig class TestPublisher(unittest.TestCase): diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 9c9cd42..ed5dabc 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -173,8 +173,8 @@ def test_update_product_base_catalog(self): } ], } - import tempfile import json as _json + import tempfile with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) @@ -211,9 +211,9 @@ def test_update_variable_base_catalog(self): } ], } - import tempfile import json as _json import os + import tempfile with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) @@ -404,9 +404,9 @@ def test_update_deepesdl_collection(self): } ], } - import tempfile import json as _json import os + import tempfile with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) diff --git a/deep_code/tests/utils/test_github_automation.py b/deep_code/tests/utils/test_github_automation.py index 671adb3..25ca33b 100644 --- a/deep_code/tests/utils/test_github_automation.py +++ b/deep_code/tests/utils/test_github_automation.py @@ -47,6 +47,7 @@ def test_clone_sync_repository_new(self, mock_run): """ No .git directory → we clone and then ensure upstream remote gets added. """ + # Simulate: "git remote -v" returns nothing so we add 'upstream' def run_side_effect(args, cwd, check, capture_output=False, text=True): if args[:3] == ["git", "remote", "-v"]: @@ -55,8 +56,9 @@ def run_side_effect(args, cwd, check, capture_output=False, text=True): mock_run.side_effect = run_side_effect - with patch.object(Path, "mkdir") as _mk, patch( - "pathlib.Path.exists", side_effect=lambda p=None: False + with ( + patch.object(Path, "mkdir") as _mk, + patch("pathlib.Path.exists", side_effect=lambda p=None: False), ): self.gha.clone_sync_repository() @@ -239,9 +241,10 @@ def test_create_branch(self, mock_run): @patch("subprocess.run") def test_add_file(self, mock_run): mock_run.return_value = make_cp() - with patch.object(Path, "mkdir") as _mk, patch.object( - Path, "write_text" - ) as _wt: + with ( + patch.object(Path, "mkdir") as _mk, + patch.object(Path, "write_text") as _wt, + ): # Ensure .git exists with patch( "pathlib.Path.exists", diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index 7264fb6..d39fd07 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -4,14 +4,13 @@ # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. -from typing import Optional import yaml class TemplateGenerator: @staticmethod - def generate_workflow_template(output_path: Optional[str] = None) -> str: + def generate_workflow_template(output_path: str | None = None) -> str: """Generate a complete template with all possible keys and placeholder values""" workflow_template = { @@ -58,7 +57,7 @@ def generate_workflow_template(output_path: Optional[str] = None) -> str: ) @staticmethod - def generate_dataset_template(output_path: Optional[str] = None) -> str: + def generate_dataset_template(output_path: str | None = None) -> str: """Generate a complete dataset template with all possible keys and placeholder values""" required = { diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index 93b9e45..07c82c3 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -94,6 +94,6 @@ def generate_prr_collection( sci_citation=config.get("sci_citation"), ) - out_dir = output_dir or config.get("prr_output_dir") or f"prr/{collection_id}" + out_dir = output_dir or config.get("prr_output_dir") or "prr" generator.save_prr_collection(out_dir) return out_dir diff --git a/deep_code/tools/test.py b/deep_code/tools/test.py index 5bdf092..b42e21f 100644 --- a/deep_code/tools/test.py +++ b/deep_code/tools/test.py @@ -1,2 +1,2 @@ -""" Execute the application package of a published experiment on a subset of input data +"""Execute the application package of a published experiment on a subset of input data to verify the reproducibility is achieved""" diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index a2c7a4c..51f4433 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -12,6 +12,7 @@ import numpy as np import pandas as pd import pyproj +import xarray as xr from pystac import ( Asset, Catalog, @@ -23,7 +24,6 @@ SpatialExtent, TemporalExtent, ) -import xarray as xr from deep_code.constants import ( CONTACTS_SCHEMA_URI, @@ -70,6 +70,7 @@ def __init__( workflow_id: str, workflow_title: str, license_type: str, + osc_project: str, access_link_root: str | None = None, documentation_link: str | None = None, osc_status: str = "ongoing", @@ -77,8 +78,7 @@ def __init__( osc_themes: list[str] | None = None, osc_missions: list[str] | None = None, cf_params: list[dict[str, Any]] | None = None, - osc_project: str = "deep-earth-system-data-lab", - osc_project_title: str = "DeepESDL", + osc_project_title: str = None, osc_project_url: str | None = None, visualisation_link: str | None = None, description: str | None = None, @@ -102,7 +102,7 @@ def __init__( self.workflow_title = workflow_title self.license_type = license_type self.osc_project = osc_project - self.osc_project_title = osc_project_title + self.osc_project_title = osc_project_title or osc_project self.osc_project_url = osc_project_url self.access_link_root = access_link_root or "s3://deep-esdl-public/" self.documentation_link = documentation_link diff --git a/deep_code/utils/helper.py b/deep_code/utils/helper.py index 9452b81..a27a786 100644 --- a/deep_code/utils/helper.py +++ b/deep_code/utils/helper.py @@ -1,6 +1,5 @@ import logging import os -from typing import Optional import xarray as xr from xcube.core.store import new_data_store @@ -25,8 +24,8 @@ def serialize(obj): def open_dataset( dataset_id: str, root: str = "deep-esdl-public", - storage_configs: Optional[list[dict]] = None, - logger: Optional[logging.Logger] = None, + storage_configs: list[dict] | None = None, + logger: logging.Logger | None = None, ) -> xr.Dataset: """Open an xarray dataset from a specified store. @@ -63,10 +62,15 @@ def open_dataset( "root": os.environ.get("S3_USER_STORAGE_BUCKET", root), "storage_options": { "anon": False, - **({ - "key": os.environ["S3_USER_STORAGE_KEY"], - "secret": os.environ["S3_USER_STORAGE_SECRET"], - } if os.environ.get("S3_USER_STORAGE_KEY") and os.environ.get("S3_USER_STORAGE_SECRET") else {}), + **( + { + "key": os.environ["S3_USER_STORAGE_KEY"], + "secret": os.environ["S3_USER_STORAGE_SECRET"], + } + if os.environ.get("S3_USER_STORAGE_KEY") + and os.environ.get("S3_USER_STORAGE_SECRET") + else {} + ), }, }, }, diff --git a/deep_code/utils/ogc_api_record.py b/deep_code/utils/ogc_api_record.py index 9b5c97c..23eecf7 100644 --- a/deep_code/utils/ogc_api_record.py +++ b/deep_code/utils/ogc_api_record.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple +from typing import Any from urllib.parse import quote, urlencode, urlparse from xrlint.util.constructible import MappingConstructible @@ -152,7 +152,7 @@ def build_child_link_to_related_experiment( } ] - def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> List[Dict[str, Any]]: + def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> list[dict[str, Any]]: return [ { "rel": "application", @@ -171,7 +171,7 @@ def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> List[Dict[str, An ] @staticmethod - def _parse_github_notebook_url(url: str) -> Tuple[str, str, str, str]: + def _parse_github_notebook_url(url: str) -> tuple[str, str, str, str]: """ Returns (repo_url, repo_name, branch, file_path_in_repo) from a GitHub URL. @@ -257,8 +257,8 @@ def __init__( links: list[dict], linkTemplates: list = [], conformsTo: list[str] = None, - geometry: Optional[Any] = None, - themes: Optional[Any] = None, + geometry: Any | None = None, + themes: Any | None = None, ): if conformsTo is None: conformsTo = [ @@ -332,7 +332,7 @@ def __init__( links: list[dict], linkTemplates=None, conformsTo: list[str] = None, - geometry: Optional[Any] = None, + geometry: Any | None = None, ): if linkTemplates is None: linkTemplates = [] diff --git a/deep_code/utils/ogc_record_generator.py b/deep_code/utils/ogc_record_generator.py index 83bd08a..e300a60 100644 --- a/deep_code/utils/ogc_record_generator.py +++ b/deep_code/utils/ogc_record_generator.py @@ -16,26 +16,24 @@ class OSCWorkflowOGCApiRecordGenerator: - """Generates OGC API record for a workflow - """ + """Generates OGC API record for a workflow""" @staticmethod def build_contact_objects(contacts_list: list[dict]) -> list[Contact]: """Build a list of Contact objects from a list of contact dictionaries. - Uses the inherited MappingConstructible logic to parse each dict. + Uses the inherited MappingConstructible logic to parse each dict. - Args: - contacts_list: A list of dictionaries, each containing contact information. + Args: + contacts_list: A list of dictionaries, each containing contact information. - Returns: - A list of Contact instances. - """ + Returns: + A list of Contact instances. + """ return [Contact.from_value(cdict) for cdict in contacts_list] @staticmethod def build_theme(osc_themes: list[str]) -> Theme: - """Convert each string into a ThemeConcept - """ + """Convert each string into a ThemeConcept""" concepts = [ThemeConcept(id=theme_str) for theme_str in osc_themes] return Theme(concepts=concepts, scheme=OSC_THEME_SCHEME) diff --git a/docs/cli.md b/docs/cli.md index 7b6da7a..6c5c89b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -61,11 +61,12 @@ needs no GitHub credentials or S3 write access (it only reads the Zarr store). I produces a self-contained `Collection → Item → Assets` tree: ``` -prr// -├── collection.json # STAC Collection (root, relative links) -└── / - └── items/ - └── .json # datacube Item covering the whole Zarr store +prr/ +└── {collection_id}/ + └── collection.json # STAC Collection (root) + └── items + └── {item_id_0}.json # datacube Item (whole Zarr) + └── {item_id_1}.json # datacube Item (whole Zarr) ``` - The **Item** carries the `datacube` extension (`cube:dimensions` / `cube:variables` @@ -83,5 +84,4 @@ still runs but logs a warning listing what is needed for full conformance. See Options: -- `--output-dir/-o`: directory to write the tree into. Defaults to `prr_output_dir` - from the config, then `prr/`. +- `--output-dir/-o`: directory to write the tree into. Defaults to `prr`. diff --git a/docs/configuration.md b/docs/configuration.md index 965ef14..cd4020d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -7,7 +7,7 @@ deep-code generate-config # writes to current directory deep-code generate-config -o ./configs # custom output folder ``` -This creates `dataset_config.yaml` and `workflow_config.yaml` with all supported fields and placeholder values. Fill them in, then run [`deep-code publish`](cli.md#publish-metadata). +This creates `dataset.yaml` and `workflow.yaml` with all supported fields and placeholder values. Fill them in, then run [`deep-code publish`](cli.md#publish-metadata). The sections below document every field in those templates. @@ -22,6 +22,8 @@ stac_catalog_s3_root: s3://bucket/stac/your-collection/ items_config: - dataset_id: your-dataset.zarr item_id: your-item # no spaces — use hyphens +osc_project: osc-project-name +osc_project_url: osc-project-url # Optional osc_themes: [cryosphere] # must match slugs at opensciencedata.esa.int/themes/catalog — auto-lowercased @@ -29,8 +31,7 @@ osc_region: global dataset_status: completed # ongoing | completed | planned (default: ongoing) documentation_link: https://example.com/docs visualisation_link: https://example.com/viewer # URL to a visualisation of the dataset -osc_project: deep-earth-system-data-lab # defaults to deep-earth-system-data-lab -access_link: s3://bucket/your-dataset.zarr # defaults to s3://deep-esdl-public/{dataset_id} +access_link_root: s3://bucket/ # defaults to s3://deep-esdl-public # CF parameter overrides (list of {name, units, ...} dicts) cf_parameter: @@ -50,23 +51,23 @@ prr_output_dir: ./prr/your-collection ### Field reference -| Field | Required | Description | -|------------------------|---|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `collection_id` | Yes | Unique ID for the STAC collection in the OSC catalog. **Must not contain spaces** — use hyphens as word separators (e.g. `My-Collection-2024`). | -| `license_type` | Yes | SPDX license identifier (e.g. `CC-BY-4.0`). Publishing fails if this field is absent. | -| `items_config` | Yes | List of `{dataset_id, item_id}` entries. Use one entry for `publish` today; the generator can emit multiple items when more are provided. | -| `dataset_id` | Legacy | Single-item fallback when `items_config` is omitted. | -| `item_id` | Legacy | Single-item fallback when `items_config` is omitted. If omitted there, deep-code falls back to the `collection_id` for compatibility. | -| `osc_themes` | No | List of OSC theme slugs (e.g. `[cryosphere, oceans]`). Values are automatically lowercased so `Land` and `land` are equivalent. | -| `osc_region` | No | Geographical region label (default: `Global`). | -| `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). | -| `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. | -| `description` | No | Human-readable description of the dataset. Overrides the `description` attribute in the Zarr store; falls back to `"No description available."` if neither is set. | -| `documentation_link` | No | URL to dataset documentation. | -| `visualisation_link` | No | URL to a visualisation of the dataset (e.g. xcube Viewer, WMS). Added as a `visualisation` link with title `"Dataset visualisation"`. | -| `osc_project` | No | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). Defaults to `deep-earth-system-data-lab`. | -| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | -| `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | +| Field | Required | Description | +|------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `collection_id` | Yes | Unique ID for the STAC collection in the OSC catalog. **Must not contain spaces** — use hyphens as word separators (e.g. `My-Collection-2024`). | +| `license_type` | Yes | SPDX license identifier (e.g. `CC-BY-4.0`). Publishing fails if this field is absent. | +| `items_config` | Yes | List of `{dataset_id, item_id}` entries. Use one entry for `publish` today; the generator can emit multiple items when more are provided. | +| `osc_project` | Yes | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). | +| `osc_project_url` | Yes | OSC project url used to link to project. | +| `osc_themes` | No | List of OSC theme slugs (e.g. `[cryosphere, oceans]`). Values are automatically lowercased so `Land` and `land` are equivalent. | +| `osc_region` | No | Geographical region label (default: `Global`). | +| `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). | +| `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. | +| `description` | No | Human-readable description of the dataset. Overrides the `description` attribute in the Zarr store; falls back to `"No description available."` if neither is set. | +| `documentation_link` | No | URL to dataset documentation. | +| `visualisation_link` | No | URL to a visualisation of the dataset (e.g. xcube Viewer, WMS). Added as a `visualisation` link with title `"Dataset visualisation"`. | +| `osc_project` | No | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). Defaults to `deep-earth-system-data-lab`. | +| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | +| `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | > The fields below are only read by [`deep-code generate-prr-collection`](cli.md#generate-a-prr-collection); `publish` ignores them. "PRR-required" means the field is required by the [PRR specification](https://eoresults.esa.int/prr_collection_specifications.html), not by the command (which still runs and warns). @@ -121,11 +122,12 @@ then the boto3 default chain (IAM role, `~/.aws/credentials`). self-contained STAC tree to a **local** directory (no S3 write, no GitHub PR): ``` -prr/your-collection/ -├── collection.json # STAC Collection (root, relative links) -└── your-collection/ - └── items/ - └── your-item.json # datacube Item covering the full Zarr store +prr/ +└── {collection_id}/ + └── collection.json # STAC Collection (root) + └── items + └── {item_id_0}.json # datacube Item (whole Zarr) + └── {item_id_1}.json # datacube Item (whole Zarr) ``` - **Collection** — declares the OSC, Scientific, Processing, Themes and CF extensions, @@ -191,4 +193,4 @@ links: | `contact` | No | List of contact objects with `name`, `organization`, and `links`. | | `links` | No | Additional OGC API record links (e.g. `related`, `describedby`). | -More templates and examples live in `dataset_config.yaml`, `workflow_config.yaml`, and `example-config/`. +More templates and examples live in `dataset.yaml`, `workflow.yaml`, and `example-config/`. diff --git a/docs/examples.md b/docs/examples.md index bf3a596..c1eac29 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,5 +1,5 @@ # Examples -- Templates: `dataset_config.yaml`, `workflow_config.yaml` +- Templates: `dataset.yaml`, `workflow.yaml` - Example configs: `examples/example-config/` - Notebooks on publishing: `examples/notebooks` diff --git a/docs/python-api.md b/docs/python-api.md index e650ba0..2203e93 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -119,7 +119,7 @@ file_dict = generator.build_zarr_stac_catalog_file_dict( ) # file_dict contains: # "s3://bucket/stac/my-collection/catalog.json" -# "s3://bucket/stac/my-collection/my-collection/items/my-item.json" +# "s3://bucket/stac/my-collection/my-collection/item.json" ``` See [STAC Catalog on S3](configuration.md#stac-catalog-on-s3) for details on the From 7edfc00d8c9de9373c0b2d62ec83608aff1bb259 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Fri, 14 Aug 2026 08:23:37 +0200 Subject: [PATCH 13/14] CI fixed --- deep_code/tests/tools/test_publish.py | 4 ++-- .../tests/utils/test_dataset_stac_generator.py | 8 ++++++++ deep_code/tools/new.py | 4 ++-- deep_code/utils/dataset_stac_generator.py | 4 ++-- deep_code/utils/ogc_api_record.py | 18 ++++++++++-------- examples/notebooks/publish_to_EarthCODE.ipynb | 10 +++++----- 6 files changed, 29 insertions(+), 19 deletions(-) diff --git a/deep_code/tests/tools/test_publish.py b/deep_code/tests/tools/test_publish.py index a411779..4048b09 100644 --- a/deep_code/tests/tools/test_publish.py +++ b/deep_code/tests/tools/test_publish.py @@ -513,7 +513,7 @@ def _setup_workflow_mocks(self): @patch("deep_code.tools.publish.LinksBuilder") @patch("deep_code.tools.publish.OSCWorkflowOGCApiRecordGenerator") def test_generate_workflow_records_mode_workflow(self, MockRG, MockLinks, MockWF): - mock_rg, mock_props, mock_wf_record, _ = self._setup_workflow_mocks() + mock_rg, _mock_props, mock_wf_record, _ = self._setup_workflow_mocks() MockRG.return_value = mock_rg MockWF.return_value = mock_wf_record @@ -537,7 +537,7 @@ def test_generate_workflow_records_mode_workflow(self, MockRG, MockLinks, MockWF def test_generate_workflow_records_mode_all( self, MockRG, MockLinks, MockWF, MockExp ): - mock_rg, mock_props, mock_wf_record, mock_exp_record = ( + mock_rg, _mock_props, mock_wf_record, mock_exp_record = ( self._setup_workflow_mocks() ) MockRG.return_value = mock_rg diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index ed5dabc..9b0a2d7 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -88,6 +88,7 @@ def setUp(self): access_link_root="s3://mock-bucket/", documentation_link="https://example.com/docs", license_type="proprietary", + osc_project="deep-earth-system-data-lab", osc_status="ongoing", osc_region="Global", osc_themes=["climate", "environment"], @@ -549,6 +550,7 @@ def test_build_zarr_stac_catalog_file_dict_multiple_items(self, mock_open_ds): workflow_id="dummy", workflow_title="test", license_type="proprietary", + osc_project="deep-earth-system-data-lab", ) file_dict = gen.build_zarr_stac_catalog_file_dict( @@ -688,6 +690,7 @@ def _make_generator(mock_ds, collection_id="my-collection", **kwargs): with patch( "deep_code.utils.dataset_stac_generator.open_dataset", return_value=mock_ds ): + kwargs.setdefault("osc_project", "deep-earth-system-data-lab") return OscDatasetStacGenerator( collection_id=collection_id, items_config=[ @@ -739,6 +742,7 @@ def test_collection_id_with_space_raises(self, mock_open_ds): workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", ) @patch("deep_code.utils.dataset_stac_generator.open_dataset") @@ -897,6 +901,7 @@ def setUp(self): workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", access_link_root="s3://bucket", osc_status="ongoing", osc_region="Global", @@ -929,6 +934,7 @@ def test_get_epsg_default_4326(self, mock_open_ds): workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", ) self.assertEqual(gen._get_crs(ds).to_epsg(), 4326) @@ -1032,6 +1038,7 @@ def test_build_prr_collection_conformant_fields(self): osc_region="Global", osc_themes=["oceans"], osc_missions=["sentinel-3"], + osc_project="deep-earth-system-data-lab", osc_project_description="A detailed project description.", osc_project_website="https://project.example.org", osc_contract_number="4000114410/15/NL/BW", @@ -1098,6 +1105,7 @@ def test_build_prr_collection_no_themes(self, mock_open_ds): workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", access_link_root="s3://bucket", ) coll = gen.build_prr_collection() diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index d39fd07..54bae9b 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -70,8 +70,8 @@ def generate_dataset_template(output_path: str | None = None) -> str: "item_id": "[REQUIRED: unique STAC item id, no spaces — use hyphens]", } ], - "osc_project": "[OPTIONAL: OSC project ID (e.g. deep-earth-system-data-lab). Defaults to deep-earth-system-data-lab]", - "osc_project_url": "[OPTIONAL: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection.]", + "osc_project": "[REQUIRED: OSC project ID (e.g. deep-earth-system-data-lab)]", + "osc_project_url": "[REQUIRED: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection.]", } optional = { diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 51f4433..9a3df7d 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -78,7 +78,7 @@ def __init__( osc_themes: list[str] | None = None, osc_missions: list[str] | None = None, cf_params: list[dict[str, Any]] | None = None, - osc_project_title: str = None, + osc_project_title: str | None = None, osc_project_url: str | None = None, visualisation_link: str | None = None, description: str | None = None, @@ -245,7 +245,7 @@ def get_variable_ids(self, dataset: xr.Dataset) -> list[str]: def get_variables_metadata(self, dataset: xr.Dataset) -> dict[str, dict]: """Extract metadata for all variables in the dataset.""" variables_metadata = {} - for var_name, variable in dataset.data_vars.items(): + for variable in dataset.data_vars.values(): var_metadata = self.extract_metadata_for_variable(variable) variables_metadata[var_metadata.get("variable_id")] = var_metadata return variables_metadata diff --git a/deep_code/utils/ogc_api_record.py b/deep_code/utils/ogc_api_record.py index 23eecf7..9d80152 100644 --- a/deep_code/utils/ogc_api_record.py +++ b/deep_code/utils/ogc_api_record.py @@ -72,13 +72,13 @@ def __init__( description: str, osc_project: str, jupyter_kernel_info: JupyterKernelInfo = None, - osc_workflow: str = None, - updated: str = None, - contacts: list[Contact] = None, - themes: list[Theme] = None, + osc_workflow: str | None = None, + updated: str | None = None, + contacts: list[Contact] | None = None, + themes: list[Theme] | None = None, keywords: list[str] | None = None, formats: list[dict] | None = None, - license: str = None, + license: str | None = None, ): self.created = created self.updated = updated @@ -255,11 +255,13 @@ def __init__( jupyter_notebook_url: str, properties: RecordProperties, links: list[dict], - linkTemplates: list = [], - conformsTo: list[str] = None, + linkTemplates: list | None = None, + conformsTo: list[str] | None = None, geometry: Any | None = None, themes: Any | None = None, ): + if linkTemplates is None: + linkTemplates = [] if conformsTo is None: conformsTo = [ OGC_API_RECORD_SPEC, @@ -331,7 +333,7 @@ def __init__( properties: RecordProperties, links: list[dict], linkTemplates=None, - conformsTo: list[str] = None, + conformsTo: list[str] | None = None, geometry: Any | None = None, ): if linkTemplates is None: diff --git a/examples/notebooks/publish_to_EarthCODE.ipynb b/examples/notebooks/publish_to_EarthCODE.ipynb index eb8d520..2fec356 100644 --- a/examples/notebooks/publish_to_EarthCODE.ipynb +++ b/examples/notebooks/publish_to_EarthCODE.ipynb @@ -234,11 +234,11 @@ "team_store = new_data_store(\n", " \"s3\", \n", " root=S3_USER_STORAGE_BUCKET, \n", - " storage_options=dict(\n", - " anon=False, \n", - " key=S3_USER_STORAGE_KEY, \n", - " secret=S3_USER_STORAGE_SECRET\n", - " )\n", + " storage_options={\n", + " 'anon': False, \n", + " 'key': S3_USER_STORAGE_KEY, \n", + " 'secret': S3_USER_STORAGE_SECRET\n", + " }\n", ")" ] }, From 5ac982fd337cc60d0e103a5779b5a966cf15d63b Mon Sep 17 00:00:00 2001 From: konstntokas Date: Mon, 17 Aug 2026 14:45:59 +0200 Subject: [PATCH 14/14] add collection title --- deep_code/tools/new.py | 1 + deep_code/tools/prr.py | 1 + deep_code/utils/dataset_stac_generator.py | 10 ++++++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index 54bae9b..5558501 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -85,6 +85,7 @@ def generate_dataset_template(output_path: str | None = None) -> str: "visualisation_link": "[OPTIONAL: URL to a visualisation of the dataset (e.g. xcube Viewer, WMS)]", "osc_project_title": "[OPTIONAL: display title of the OSC project as it appears in the catalog (e.g. DeepESDL). Defaults to a formatted version of osc_project if omitted]", "access_link_root": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public]", + "collection_title": "[OPTIONAL: title present in the collection and in the STAC browser]", "cf_parameter": [ {"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"} ], diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index 07c82c3..6a65035 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -72,6 +72,7 @@ def generate_prr_collection( workflow_title=config.get("workflow_title") or "", license_type=license_type, documentation_link=config.get("documentation_link"), + collection_title=config.get("collection_title"), access_link_root=config.get("access_link_root"), osc_status=config.get("dataset_status") or "ongoing", osc_region=config.get("osc_region") or "Global", diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 9a3df7d..373029b 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -53,6 +53,7 @@ class OscDatasetStacGenerator: collection_id: Unique identifier for the STAC collection. items_config: List of item configuration entries. Each item maps one dataset_id to one item_id + collection_title: Title present in the collection and in the STAC browser access_link_root: Public access link to the root of the datasets. documentation_link: Link to dataset documentation. osc_status: Status of the dataset (e.g., "ongoing"). @@ -71,6 +72,7 @@ def __init__( workflow_title: str, license_type: str, osc_project: str, + collection_title: str | None = None, access_link_root: str | None = None, documentation_link: str | None = None, osc_status: str = "ongoing", @@ -105,10 +107,14 @@ def __init__( self.osc_project_title = osc_project_title or osc_project self.osc_project_url = osc_project_url self.access_link_root = access_link_root or "s3://deep-esdl-public/" + self.collection_title = collection_title or collection_id self.documentation_link = documentation_link self.osc_status = osc_status self.osc_region = osc_region - self.osc_themes = [t.lower() for t in (osc_themes or [])] + if osc_themes is None: + osc_themes = [] + assert isinstance(osc_themes, list) + self.osc_themes = [t.lower() for t in osc_themes] self.osc_missions = osc_missions or [] self.cf_params = cf_params or {} self.visualisation_link = visualisation_link @@ -966,7 +972,7 @@ def build_prr_collection(self) -> Collection: description=self.description or "No description provided.", extent=Extent(spatial=spatial_extent, temporal=temporal_extent), license=self.license_type, - title=self.collection_id, + title=self.collection_title, ) collection.stac_version = "1.0.0"