diff --git a/scripts/cli/cli_helper.py b/scripts/cli/cli_helper.py index ae0fa642..1ec377fa 100644 --- a/scripts/cli/cli_helper.py +++ b/scripts/cli/cli_helper.py @@ -101,12 +101,6 @@ def valid_date(s: str) -> datetime: raise argparse.ArgumentTypeError(msg) from e -def nullable_str(s: str) -> Optional[str]: - if s == "": - return None - return s - - def parse_list(list_s: str, separator: Optional[str] = ";") -> List[str]: """Transform a string representing a list to a list of strings example: "foo; bar; foo bar" -> ["foo", "bar", "foo bar"] diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py index d97d032e..64007b96 100644 --- a/scripts/collection_from_items.py +++ b/scripts/collection_from_items.py @@ -6,18 +6,23 @@ from boto3 import client from linz_logger import get_log -from scripts.cli.cli_helper import coalesce_multi_single, nullable_str, valid_date +from scripts.cli.cli_helper import coalesce_multi_single, valid_date from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_json_in_uri from scripts.logging.time_helper import time_in_ms from scripts.stac.imagery.collection import ImageryCollection -from scripts.stac.imagery.generate_metadata import ElevationSubtypes, ImagerySubtypes, generate_description, generate_title +from scripts.stac.imagery.generate_metadata import ( + HUMAN_READABLE_REGIONS, + ElevationSubtypes, + ImagerySubtypes, + generate_description, + generate_title, +) from scripts.stac.imagery.provider import Provider, ProviderRole # pylint: disable-msg=too-many-locals def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument("--preset", dest="preset", required=True, help="Standardised file format. Example: webp") parser.add_argument("--uri", dest="uri", help="s3 path to items and collection.json write location", required=True) parser.add_argument("--collection-id", dest="collection_id", help="Collection ID", required=True) parser.add_argument( @@ -25,64 +30,35 @@ def main() -> None: dest="subtype", help="Dataset subtype description", required=True, - choices=[ - ImagerySubtypes.AERIAL, - ImagerySubtypes.HISTORICAL, - ImagerySubtypes.SATELLIE, - ImagerySubtypes.URBAN, - ImagerySubtypes.RURAL, - ElevationSubtypes.DEM, - ElevationSubtypes.DSM, - ], + choices=[list(ImagerySubtypes), list(ElevationSubtypes)], ) parser.add_argument( "--region", dest="region", help="Region of Dataset", required=True, - choices=[ - "antarctica", - "auckland", - "bay-of-plenty", - "canterbury", - "gisborne", - "global", - "hawkes-bay", - "manawatu-whanganui", - "marlborough", - "nelson", - "new-zealand", - "northland", - "otago", - "pacific-islands", - "southland", - "taranaki", - "tasman", - "waikato", - "wellington", - "west-coast", - ], + choices=HUMAN_READABLE_REGIONS.keys(), ) - parser.add_argument("--gsd", dest="gsd", help="GSD of imagery Dataset", type=nullable_str, required=True) + parser.add_argument("--gsd", dest="gsd", help="GSD of imagery Dataset", type=str, required=True) parser.add_argument( - "--location", dest="location", help="Optional Location of dataset, e.g.- Hutt City", type=nullable_str, required=False + "--location", dest="location", help="Optional Location of dataset, e.g.- Hutt City", type=str, required=False ) parser.add_argument( "--start-date", dest="start_date", - help="Start datetime in format YYYY-MM-DD (Inclusive)", + help="Start date in format YYYY-MM-DD (Inclusive)", type=valid_date, required=True, ) parser.add_argument( - "--end-date", dest="end_date", help="End datetime in format YYYY-MM-DD (Inclusive)", type=valid_date, required=True + "--end-date", dest="end_date", help="End date in format YYYY-MM-DD (Inclusive)", type=valid_date, required=True ) - parser.add_argument("--event", dest="dest", help="Event name if applicable", type=nullable_str, required=False) + parser.add_argument("--event", dest="dest", help="Event name if applicable", type=str, required=False) parser.add_argument( "--historic-survey-number", dest="historic_survey_number", help="Historic Survey Number if Applicable. E.g.- SCN8844", - type=nullable_str, + type=str, required=False, ) parser.add_argument( diff --git a/scripts/stac/imagery/generate_metadata.py b/scripts/stac/imagery/generate_metadata.py index c4d55de2..00ff5e49 100644 --- a/scripts/stac/imagery/generate_metadata.py +++ b/scripts/stac/imagery/generate_metadata.py @@ -1,4 +1,3 @@ -import string from datetime import datetime from enum import Enum from typing import Optional @@ -10,7 +9,7 @@ def __init__(self, subtype: str) -> None: class ImagerySubtypes(str, Enum): - SATELLIE = "Satellite Imagery" + SATELLITE = "Satellite Imagery" URBAN = "Urban Aerial Photos" RURAL = "Rural Aerial Photos" AERIAL = "Aerial Photos" @@ -22,6 +21,30 @@ class ElevationSubtypes(str, Enum): DSM = "DSM" +HUMAN_READABLE_REGIONS = { + "antarctica": "Antarctica", + "auckland": "Auckland", + "bay-of-plenty": "Bay of Plenty", + "canterbury": "Canterbury", + "gisborne": "Gisborne", + "global": "Global", + "hawkes-bay": "Hawke's Bay", + "manawatu-whanganui": "Manawatū-Whanganui", + "marlborough": "Marlborough", + "nelson": "Nelson", + "new-zealand": "New Zealand", + "northland": "Northland", + "otago": "Otago", + "pacific-islands": "Pacific Islands", + "southland": "Southland", + "taranaki": "Taranaki", + "tasman": "Tasman", + "waikato": "Waikato", + "wellington": "Wellington", + "west-coast": "West Coast", +} + + def generate_title( subtype: str, region: str, @@ -57,13 +80,13 @@ def generate_title( # pylint: disable-msg=too-many-arguments date = _format_date_for_metadata(start_datetime, end_datetime) - name = _format_name_for_title(_region_map(region), location) + name = _format_name_for_title(HUMAN_READABLE_REGIONS[region], location) preview = _is_preview(lifecycle) if historic_survey_number: return " ".join(f"{name} {gsd} {historic_survey_number} ({date}) {preview or ''}".split()) - if subtype in [ImagerySubtypes.SATELLIE, ImagerySubtypes.URBAN, ImagerySubtypes.RURAL]: + if subtype in [ImagerySubtypes.SATELLITE, ImagerySubtypes.URBAN, ImagerySubtypes.RURAL]: return " ".join(f"{name} {gsd} {event or ''} {subtype} ({date}) {preview or ''}".split()) if subtype in [ElevationSubtypes.DEM, ElevationSubtypes.DSM]: return " ".join( @@ -106,11 +129,11 @@ def generate_description( """ date = _format_date_for_metadata(start_date, end_date) location_txt = _format_location_for_elevation_description(location) # TODO: rename - region = _region_map(region) + region = HUMAN_READABLE_REGIONS[region] if historic_survey_number: desc = f"Scanned aerial imagery within the {region} region captured in {date}" - elif subtype == ImagerySubtypes.SATELLIE: + elif subtype == ImagerySubtypes.SATELLITE: desc = f"Satellite imagery within the {region} region captured in {date}" elif subtype in [ImagerySubtypes.URBAN, ImagerySubtypes.RURAL]: desc = f"Orthophotography within the {region} region captured in the {date} flying season" @@ -156,12 +179,3 @@ def _format_event_for_elevation_title(event: Optional[str]) -> Optional[str]: if event: return f"- {event}" return None - - -def _region_map(region: str) -> str: - """Convert region parameters from ascii input to uft-8""" - if region == "hawkes-bay": - return "Hawke's Bay" - if region == "manawatu-whanganui": - return "Manawatū-Whanganui" - return string.capwords(region.replace("-", " ")) diff --git a/scripts/stac/tests/generate_description_test.py b/scripts/stac/tests/generate_description_test.py index 95d30d74..c94f3d4b 100644 --- a/scripts/stac/tests/generate_description_test.py +++ b/scripts/stac/tests/generate_description_test.py @@ -5,34 +5,34 @@ def test_generate_description_imagery() -> None: description = "Orthophotography within the Auckland region captured in the 2023 flying season." - generated_description = generate_description("Rural Aerial Photos", "Auckland", datetime(2023, 1, 1), datetime(2023, 2, 2)) + generated_description = generate_description("Rural Aerial Photos", "auckland", datetime(2023, 1, 1), datetime(2023, 2, 2)) assert generated_description == description def test_generate_description_elevation() -> None: description = "Digital Elevation Model within the Auckland region in 2023." - generated_description = generate_description("DEM", "Auckland", datetime(2023, 1, 1), datetime(2023, 2, 2)) + generated_description = generate_description("DEM", "auckland", datetime(2023, 1, 1), datetime(2023, 2, 2)) assert generated_description == description def test_generate_description_elevation_location_input() -> None: description = "Digital Elevation Model within the Auckland - Central region in 2023." generated_description = generate_description( - "DEM", "Auckland", datetime(2023, 1, 1), datetime(2023, 2, 2), location="Central" + "DEM", "auckland", datetime(2023, 1, 1), datetime(2023, 2, 2), location="Central" ) assert generated_description == description def test_generate_description_satellite_imagery() -> None: description = "Satellite imagery within the Auckland region captured in 2023." - generated_description = generate_description("Satellite Imagery", "Auckland", datetime(2023, 1, 1), datetime(2023, 2, 2)) + generated_description = generate_description("Satellite Imagery", "auckland", datetime(2023, 1, 1), datetime(2023, 2, 2)) assert generated_description == description def test_generate_description_historic_imagery() -> None: description = "Scanned aerial imagery within the Auckland region captured in 2000." generated_description = generate_description( - "Aerial Photos", "Auckland", datetime(2000, 1, 1), datetime(2000, 2, 2), historic_survey_number="SNC8844" + "Aerial Photos", "auckland", datetime(2000, 1, 1), datetime(2000, 2, 2), historic_survey_number="SNC8844" ) assert generated_description == description @@ -41,6 +41,6 @@ def test_generate_description_event() -> None: description = "Orthophotography within the Hawke's Bay region captured in the 2023 flying season, \ published as a record of the Cyclone Gabrielle event." generated_description = generate_description( - "Rural Aerial Photos", "Hawke's Bay", datetime(2023, 1, 1), datetime(2023, 2, 2), event="Cyclone Gabrielle" + "Rural Aerial Photos", "hawkes-bay", datetime(2023, 1, 1), datetime(2023, 2, 2), event="Cyclone Gabrielle" ) assert generated_description == description diff --git a/scripts/stac/tests/generate_title_test.py b/scripts/stac/tests/generate_title_test.py index 6e0162cf..753f92f8 100644 --- a/scripts/stac/tests/generate_title_test.py +++ b/scripts/stac/tests/generate_title_test.py @@ -6,27 +6,27 @@ def test_generate_imagery_title() -> None: title = "Auckland 0.3m Rural Aerial Photos (2023)" generated_title = generate_title( - "Rural Aerial Photos", "Auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed" + "Rural Aerial Photos", "auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed" ) assert generated_title == title def test_generate_dem_title() -> None: title = "Auckland LiDAR 0.3m DEM (2023)" - generated_title = generate_title("DEM", "Auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed") + generated_title = generate_title("DEM", "auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed") assert generated_title == title def test_generate_dsm_title() -> None: title = "Auckland LiDAR 0.3m DSM (2023)" - generated_title = generate_title("DSM", "Auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed") + generated_title = generate_title("DSM", "auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed") assert generated_title == title def test_generate_satellite_imagery_title() -> None: title = "Auckland 0.5m Satellite Imagery (2023)" generated_title = generate_title( - "Satellite Imagery", "Auckland", "0.5m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed" + "Satellite Imagery", "auckland", "0.5m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed" ) assert generated_title == title @@ -35,7 +35,7 @@ def test_generate_historic_imagery_title() -> None: title = "Auckland 0.3m SNC8844 (2000)" generated_title = generate_title( "Aerial Photos", - "Auckland", + "auckland", "0.3m", datetime(2000, 1, 1), datetime(2000, 2, 2), @@ -48,7 +48,7 @@ def test_generate_historic_imagery_title() -> None: def test_generate_title_long_date() -> None: title = "Auckland 0.3m Urban Aerial Photos (2023 - 2024)" generated_title = generate_title( - "Urban Aerial Photos", "Auckland", "0.3m", datetime(2023, 1, 1), datetime(2024, 2, 2), "completed" + "Urban Aerial Photos", "auckland", "0.3m", datetime(2023, 1, 1), datetime(2024, 2, 2), "completed" ) assert generated_title == title @@ -57,7 +57,7 @@ def test_generate_title_location() -> None: title = "Banks Penninsula 0.3m Rural Aerial Photos (2023)" generated_title = generate_title( "Rural Aerial Photos", - "Canterbury", + "canterbury", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), @@ -71,7 +71,7 @@ def test_generate_title_event_imagery() -> None: title = "Hawke's Bay 0.3m Cyclone Gabrielle Rural Aerial Photos (2023)" generated_title = generate_title( "Rural Aerial Photos", - "Hawke's Bay", + "hawkes-bay", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), @@ -84,7 +84,7 @@ def test_generate_title_event_imagery() -> None: def test_generate_title_event_elevation() -> None: title = "Hawke's Bay - Cyclone Gabrielle LiDAR 0.3m DSM (2023)" generated_title = generate_title( - "DSM", "Hawke's Bay", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed", event="Cyclone Gabrielle" + "DSM", "hawkes-bay", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed", event="Cyclone Gabrielle" ) assert generated_title == title @@ -93,7 +93,7 @@ def test_generate_title_event_satellite_imagery() -> None: title = "Hawke's Bay 0.5m Cyclone Gabrielle Satellite Imagery (2023)" generated_title = generate_title( "Satellite Imagery", - "Hawke's Bay", + "hawkes-bay", "0.5m", datetime(2023, 1, 1), datetime(2023, 2, 2), @@ -105,7 +105,7 @@ def test_generate_title_event_satellite_imagery() -> None: def test_generate_dsm_title_preview() -> None: title = "Auckland LiDAR 0.3m DSM (2023) - preview" - generated_title = generate_title("DSM", "Auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "preview") + generated_title = generate_title("DSM", "auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "preview") assert generated_title == title @@ -113,7 +113,7 @@ def test_generate_imagery_title_empty_optional_str() -> None: title = "Auckland 0.3m Rural Aerial Photos (2023)" generated_title = generate_title( "Rural Aerial Photos", - "Auckland", + "auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2),