From 1fe774522e8f2b79a762d98b56940145db1e08b5 Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Fri, 8 Dec 2023 12:05:34 +1300 Subject: [PATCH 1/8] fix: remove unused param --- scripts/collection_from_items.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py index d97d032e..ca3ef2f2 100644 --- a/scripts/collection_from_items.py +++ b/scripts/collection_from_items.py @@ -17,7 +17,6 @@ # pylint: disable-msg=too-many-locals def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument("--preset", dest="preset", required=True, help="Standardised file format. Example: webp") parser.add_argument("--uri", dest="uri", help="s3 path to items and collection.json write location", required=True) parser.add_argument("--collection-id", dest="collection_id", help="Collection ID", required=True) parser.add_argument( From ab8a8cc8c460f4f1e97bf817b059c62bdf9201c0 Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Fri, 8 Dec 2023 12:56:04 +1300 Subject: [PATCH 2/8] fix: tidy choices enum listing --- scripts/collection_from_items.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py index ca3ef2f2..c1cc3fa4 100644 --- a/scripts/collection_from_items.py +++ b/scripts/collection_from_items.py @@ -24,15 +24,7 @@ def main() -> None: dest="subtype", help="Dataset subtype description", required=True, - choices=[ - ImagerySubtypes.AERIAL, - ImagerySubtypes.HISTORICAL, - ImagerySubtypes.SATELLIE, - ImagerySubtypes.URBAN, - ImagerySubtypes.RURAL, - ElevationSubtypes.DEM, - ElevationSubtypes.DSM, - ], + choices=[list(ImagerySubtypes), list(ElevationSubtypes)], ) parser.add_argument( "--region", From 984ce9c24f7df96d397798fe14a1c327b3b2841e Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Fri, 8 Dec 2023 12:56:54 +1300 Subject: [PATCH 3/8] fix: gsd cannot be none --- scripts/collection_from_items.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py index c1cc3fa4..25c7dd68 100644 --- a/scripts/collection_from_items.py +++ b/scripts/collection_from_items.py @@ -54,7 +54,7 @@ def main() -> None: "west-coast", ], ) - parser.add_argument("--gsd", dest="gsd", help="GSD of imagery Dataset", type=nullable_str, required=True) + parser.add_argument("--gsd", dest="gsd", help="GSD of imagery Dataset", type=str, required=True) parser.add_argument( "--location", dest="location", help="Optional Location of dataset, e.g.- Hutt City", type=nullable_str, required=False ) From b7b64d67a54341c9a61e3cf95d03e649b62b65b3 Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Fri, 8 Dec 2023 13:13:31 +1300 Subject: [PATCH 4/8] fix: str='' is the same as str=None --- scripts/cli/cli_helper.py | 6 ------ scripts/collection_from_items.py | 8 ++++---- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/scripts/cli/cli_helper.py b/scripts/cli/cli_helper.py index ae0fa642..1ec377fa 100644 --- a/scripts/cli/cli_helper.py +++ b/scripts/cli/cli_helper.py @@ -101,12 +101,6 @@ def valid_date(s: str) -> datetime: raise argparse.ArgumentTypeError(msg) from e -def nullable_str(s: str) -> Optional[str]: - if s == "": - return None - return s - - def parse_list(list_s: str, separator: Optional[str] = ";") -> List[str]: """Transform a string representing a list to a list of strings example: "foo; bar; foo bar" -> ["foo", "bar", "foo bar"] diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py index 25c7dd68..ba6409da 100644 --- a/scripts/collection_from_items.py +++ b/scripts/collection_from_items.py @@ -6,7 +6,7 @@ from boto3 import client from linz_logger import get_log -from scripts.cli.cli_helper import coalesce_multi_single, nullable_str, valid_date +from scripts.cli.cli_helper import coalesce_multi_single, valid_date from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_json_in_uri from scripts.logging.time_helper import time_in_ms from scripts.stac.imagery.collection import ImageryCollection @@ -56,7 +56,7 @@ def main() -> None: ) parser.add_argument("--gsd", dest="gsd", help="GSD of imagery Dataset", type=str, required=True) parser.add_argument( - "--location", dest="location", help="Optional Location of dataset, e.g.- Hutt City", type=nullable_str, required=False + "--location", dest="location", help="Optional Location of dataset, e.g.- Hutt City", type=str, required=False ) parser.add_argument( "--start-date", @@ -68,12 +68,12 @@ def main() -> None: parser.add_argument( "--end-date", dest="end_date", help="End datetime in format YYYY-MM-DD (Inclusive)", type=valid_date, required=True ) - parser.add_argument("--event", dest="dest", help="Event name if applicable", type=nullable_str, required=False) + parser.add_argument("--event", dest="dest", help="Event name if applicable", type=str, required=False) parser.add_argument( "--historic-survey-number", dest="historic_survey_number", help="Historic Survey Number if Applicable. E.g.- SCN8844", - type=nullable_str, + type=str, required=False, ) parser.add_argument( From 76d3e0b0c6a7d724ea8700a5c5fb0c687133977c Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Fri, 8 Dec 2023 13:20:41 +1300 Subject: [PATCH 5/8] fix: typo --- scripts/stac/imagery/generate_metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/stac/imagery/generate_metadata.py b/scripts/stac/imagery/generate_metadata.py index c4d55de2..fa1662b6 100644 --- a/scripts/stac/imagery/generate_metadata.py +++ b/scripts/stac/imagery/generate_metadata.py @@ -10,7 +10,7 @@ def __init__(self, subtype: str) -> None: class ImagerySubtypes(str, Enum): - SATELLIE = "Satellite Imagery" + SATELLITE = "Satellite Imagery" URBAN = "Urban Aerial Photos" RURAL = "Rural Aerial Photos" AERIAL = "Aerial Photos" @@ -63,7 +63,7 @@ def generate_title( if historic_survey_number: return " ".join(f"{name} {gsd} {historic_survey_number} ({date}) {preview or ''}".split()) - if subtype in [ImagerySubtypes.SATELLIE, ImagerySubtypes.URBAN, ImagerySubtypes.RURAL]: + if subtype in [ImagerySubtypes.SATELLITE, ImagerySubtypes.URBAN, ImagerySubtypes.RURAL]: return " ".join(f"{name} {gsd} {event or ''} {subtype} ({date}) {preview or ''}".split()) if subtype in [ElevationSubtypes.DEM, ElevationSubtypes.DSM]: return " ".join( @@ -110,7 +110,7 @@ def generate_description( if historic_survey_number: desc = f"Scanned aerial imagery within the {region} region captured in {date}" - elif subtype == ImagerySubtypes.SATELLIE: + elif subtype == ImagerySubtypes.SATELLITE: desc = f"Satellite imagery within the {region} region captured in {date}" elif subtype in [ImagerySubtypes.URBAN, ImagerySubtypes.RURAL]: desc = f"Orthophotography within the {region} region captured in the {date} flying season" From bfbf27233da309d94ed193d1790fef6e67ad8ae6 Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Fri, 8 Dec 2023 13:42:34 +1300 Subject: [PATCH 6/8] fix: use enum dictionary to map regions --- scripts/collection_from_items.py | 31 +++++------------- scripts/stac/imagery/generate_metadata.py | 38 ++++++++++++++++------- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py index ba6409da..20619b04 100644 --- a/scripts/collection_from_items.py +++ b/scripts/collection_from_items.py @@ -10,7 +10,13 @@ from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_json_in_uri from scripts.logging.time_helper import time_in_ms from scripts.stac.imagery.collection import ImageryCollection -from scripts.stac.imagery.generate_metadata import ElevationSubtypes, ImagerySubtypes, generate_description, generate_title +from scripts.stac.imagery.generate_metadata import ( + HUMAN_READABLE_REGIONS, + ElevationSubtypes, + ImagerySubtypes, + generate_description, + generate_title, +) from scripts.stac.imagery.provider import Provider, ProviderRole @@ -31,28 +37,7 @@ def main() -> None: dest="region", help="Region of Dataset", required=True, - choices=[ - "antarctica", - "auckland", - "bay-of-plenty", - "canterbury", - "gisborne", - "global", - "hawkes-bay", - "manawatu-whanganui", - "marlborough", - "nelson", - "new-zealand", - "northland", - "otago", - "pacific-islands", - "southland", - "taranaki", - "tasman", - "waikato", - "wellington", - "west-coast", - ], + choices=HUMAN_READABLE_REGIONS.keys(), ) parser.add_argument("--gsd", dest="gsd", help="GSD of imagery Dataset", type=str, required=True) parser.add_argument( diff --git a/scripts/stac/imagery/generate_metadata.py b/scripts/stac/imagery/generate_metadata.py index fa1662b6..00ff5e49 100644 --- a/scripts/stac/imagery/generate_metadata.py +++ b/scripts/stac/imagery/generate_metadata.py @@ -1,4 +1,3 @@ -import string from datetime import datetime from enum import Enum from typing import Optional @@ -22,6 +21,30 @@ class ElevationSubtypes(str, Enum): DSM = "DSM" +HUMAN_READABLE_REGIONS = { + "antarctica": "Antarctica", + "auckland": "Auckland", + "bay-of-plenty": "Bay of Plenty", + "canterbury": "Canterbury", + "gisborne": "Gisborne", + "global": "Global", + "hawkes-bay": "Hawke's Bay", + "manawatu-whanganui": "Manawatū-Whanganui", + "marlborough": "Marlborough", + "nelson": "Nelson", + "new-zealand": "New Zealand", + "northland": "Northland", + "otago": "Otago", + "pacific-islands": "Pacific Islands", + "southland": "Southland", + "taranaki": "Taranaki", + "tasman": "Tasman", + "waikato": "Waikato", + "wellington": "Wellington", + "west-coast": "West Coast", +} + + def generate_title( subtype: str, region: str, @@ -57,7 +80,7 @@ def generate_title( # pylint: disable-msg=too-many-arguments date = _format_date_for_metadata(start_datetime, end_datetime) - name = _format_name_for_title(_region_map(region), location) + name = _format_name_for_title(HUMAN_READABLE_REGIONS[region], location) preview = _is_preview(lifecycle) if historic_survey_number: @@ -106,7 +129,7 @@ def generate_description( """ date = _format_date_for_metadata(start_date, end_date) location_txt = _format_location_for_elevation_description(location) # TODO: rename - region = _region_map(region) + region = HUMAN_READABLE_REGIONS[region] if historic_survey_number: desc = f"Scanned aerial imagery within the {region} region captured in {date}" @@ -156,12 +179,3 @@ def _format_event_for_elevation_title(event: Optional[str]) -> Optional[str]: if event: return f"- {event}" return None - - -def _region_map(region: str) -> str: - """Convert region parameters from ascii input to uft-8""" - if region == "hawkes-bay": - return "Hawke's Bay" - if region == "manawatu-whanganui": - return "Manawatū-Whanganui" - return string.capwords(region.replace("-", " ")) From 2c4c7cff2b81fd97be853cc0b9c46507e2968840 Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Fri, 8 Dec 2023 13:50:43 +1300 Subject: [PATCH 7/8] fix: update tests --- .../stac/tests/generate_description_test.py | 12 +++++----- scripts/stac/tests/generate_title_test.py | 24 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/stac/tests/generate_description_test.py b/scripts/stac/tests/generate_description_test.py index 95d30d74..c94f3d4b 100644 --- a/scripts/stac/tests/generate_description_test.py +++ b/scripts/stac/tests/generate_description_test.py @@ -5,34 +5,34 @@ def test_generate_description_imagery() -> None: description = "Orthophotography within the Auckland region captured in the 2023 flying season." - generated_description = generate_description("Rural Aerial Photos", "Auckland", datetime(2023, 1, 1), datetime(2023, 2, 2)) + generated_description = generate_description("Rural Aerial Photos", "auckland", datetime(2023, 1, 1), datetime(2023, 2, 2)) assert generated_description == description def test_generate_description_elevation() -> None: description = "Digital Elevation Model within the Auckland region in 2023." - generated_description = generate_description("DEM", "Auckland", datetime(2023, 1, 1), datetime(2023, 2, 2)) + generated_description = generate_description("DEM", "auckland", datetime(2023, 1, 1), datetime(2023, 2, 2)) assert generated_description == description def test_generate_description_elevation_location_input() -> None: description = "Digital Elevation Model within the Auckland - Central region in 2023." generated_description = generate_description( - "DEM", "Auckland", datetime(2023, 1, 1), datetime(2023, 2, 2), location="Central" + "DEM", "auckland", datetime(2023, 1, 1), datetime(2023, 2, 2), location="Central" ) assert generated_description == description def test_generate_description_satellite_imagery() -> None: description = "Satellite imagery within the Auckland region captured in 2023." - generated_description = generate_description("Satellite Imagery", "Auckland", datetime(2023, 1, 1), datetime(2023, 2, 2)) + generated_description = generate_description("Satellite Imagery", "auckland", datetime(2023, 1, 1), datetime(2023, 2, 2)) assert generated_description == description def test_generate_description_historic_imagery() -> None: description = "Scanned aerial imagery within the Auckland region captured in 2000." generated_description = generate_description( - "Aerial Photos", "Auckland", datetime(2000, 1, 1), datetime(2000, 2, 2), historic_survey_number="SNC8844" + "Aerial Photos", "auckland", datetime(2000, 1, 1), datetime(2000, 2, 2), historic_survey_number="SNC8844" ) assert generated_description == description @@ -41,6 +41,6 @@ def test_generate_description_event() -> None: description = "Orthophotography within the Hawke's Bay region captured in the 2023 flying season, \ published as a record of the Cyclone Gabrielle event." generated_description = generate_description( - "Rural Aerial Photos", "Hawke's Bay", datetime(2023, 1, 1), datetime(2023, 2, 2), event="Cyclone Gabrielle" + "Rural Aerial Photos", "hawkes-bay", datetime(2023, 1, 1), datetime(2023, 2, 2), event="Cyclone Gabrielle" ) assert generated_description == description diff --git a/scripts/stac/tests/generate_title_test.py b/scripts/stac/tests/generate_title_test.py index 6e0162cf..753f92f8 100644 --- a/scripts/stac/tests/generate_title_test.py +++ b/scripts/stac/tests/generate_title_test.py @@ -6,27 +6,27 @@ def test_generate_imagery_title() -> None: title = "Auckland 0.3m Rural Aerial Photos (2023)" generated_title = generate_title( - "Rural Aerial Photos", "Auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed" + "Rural Aerial Photos", "auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed" ) assert generated_title == title def test_generate_dem_title() -> None: title = "Auckland LiDAR 0.3m DEM (2023)" - generated_title = generate_title("DEM", "Auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed") + generated_title = generate_title("DEM", "auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed") assert generated_title == title def test_generate_dsm_title() -> None: title = "Auckland LiDAR 0.3m DSM (2023)" - generated_title = generate_title("DSM", "Auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed") + generated_title = generate_title("DSM", "auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed") assert generated_title == title def test_generate_satellite_imagery_title() -> None: title = "Auckland 0.5m Satellite Imagery (2023)" generated_title = generate_title( - "Satellite Imagery", "Auckland", "0.5m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed" + "Satellite Imagery", "auckland", "0.5m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed" ) assert generated_title == title @@ -35,7 +35,7 @@ def test_generate_historic_imagery_title() -> None: title = "Auckland 0.3m SNC8844 (2000)" generated_title = generate_title( "Aerial Photos", - "Auckland", + "auckland", "0.3m", datetime(2000, 1, 1), datetime(2000, 2, 2), @@ -48,7 +48,7 @@ def test_generate_historic_imagery_title() -> None: def test_generate_title_long_date() -> None: title = "Auckland 0.3m Urban Aerial Photos (2023 - 2024)" generated_title = generate_title( - "Urban Aerial Photos", "Auckland", "0.3m", datetime(2023, 1, 1), datetime(2024, 2, 2), "completed" + "Urban Aerial Photos", "auckland", "0.3m", datetime(2023, 1, 1), datetime(2024, 2, 2), "completed" ) assert generated_title == title @@ -57,7 +57,7 @@ def test_generate_title_location() -> None: title = "Banks Penninsula 0.3m Rural Aerial Photos (2023)" generated_title = generate_title( "Rural Aerial Photos", - "Canterbury", + "canterbury", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), @@ -71,7 +71,7 @@ def test_generate_title_event_imagery() -> None: title = "Hawke's Bay 0.3m Cyclone Gabrielle Rural Aerial Photos (2023)" generated_title = generate_title( "Rural Aerial Photos", - "Hawke's Bay", + "hawkes-bay", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), @@ -84,7 +84,7 @@ def test_generate_title_event_imagery() -> None: def test_generate_title_event_elevation() -> None: title = "Hawke's Bay - Cyclone Gabrielle LiDAR 0.3m DSM (2023)" generated_title = generate_title( - "DSM", "Hawke's Bay", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed", event="Cyclone Gabrielle" + "DSM", "hawkes-bay", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "completed", event="Cyclone Gabrielle" ) assert generated_title == title @@ -93,7 +93,7 @@ def test_generate_title_event_satellite_imagery() -> None: title = "Hawke's Bay 0.5m Cyclone Gabrielle Satellite Imagery (2023)" generated_title = generate_title( "Satellite Imagery", - "Hawke's Bay", + "hawkes-bay", "0.5m", datetime(2023, 1, 1), datetime(2023, 2, 2), @@ -105,7 +105,7 @@ def test_generate_title_event_satellite_imagery() -> None: def test_generate_dsm_title_preview() -> None: title = "Auckland LiDAR 0.3m DSM (2023) - preview" - generated_title = generate_title("DSM", "Auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "preview") + generated_title = generate_title("DSM", "auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), "preview") assert generated_title == title @@ -113,7 +113,7 @@ def test_generate_imagery_title_empty_optional_str() -> None: title = "Auckland 0.3m Rural Aerial Photos (2023)" generated_title = generate_title( "Rural Aerial Photos", - "Auckland", + "auckland", "0.3m", datetime(2023, 1, 1), datetime(2023, 2, 2), From bbe29a5e880f2e054652410681832c9fb4b0cfad Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Fri, 8 Dec 2023 13:52:46 +1300 Subject: [PATCH 8/8] fix: nit --- scripts/collection_from_items.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py index 20619b04..64007b96 100644 --- a/scripts/collection_from_items.py +++ b/scripts/collection_from_items.py @@ -46,12 +46,12 @@ def main() -> None: parser.add_argument( "--start-date", dest="start_date", - help="Start datetime in format YYYY-MM-DD (Inclusive)", + help="Start date in format YYYY-MM-DD (Inclusive)", type=valid_date, required=True, ) parser.add_argument( - "--end-date", dest="end_date", help="End datetime in format YYYY-MM-DD (Inclusive)", type=valid_date, required=True + "--end-date", dest="end_date", help="End date in format YYYY-MM-DD (Inclusive)", type=valid_date, required=True ) parser.add_argument("--event", dest="dest", help="Event name if applicable", type=str, required=False) parser.add_argument(