Merge remote-tracking branch 'origin/asc'

MDavidson17 · Aug 25, 2023 · 88017fe · 88017fe
2 parents 5df0e16 + 912675c
commit 88017fe
Show file tree

Hide file tree

Showing 8 changed files with 164 additions and 4 deletions.
diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml
@@ -0,0 +1,58 @@
+name: Format and Test
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Use Python "3.10.6"
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10.6"
+      - name: Install
+        run: |
+          pip install poetry
+          poetry install
+      - name: Format
+        run: |
+          poetry run pre-commit run --all-files
+      - name: Unit Tests
+        run: |
+          poetry run pytest .
+
+      - name: Build containers
+        run: |
+          docker build . --tag topo-imagery --label "github_run_id=${GITHUB_RUN_ID}"
+
+      - name: End to end test - Aerial Imagery
+        run: |
+          # Standardising test 
+          docker run  -v ${HOME}/tmp/:/tmp/ topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01
+          cmp --silent ${HOME}/tmp/BG35_1000_4829.tiff ./scripts/tests/data/output/BG35_1000_4829.tiff
+
+      - name: End to end test - Elevation
+        run: |
+          # Standardising test
+          docker run  -v ${HOME}/tmp/:/tmp/ topo-imagery python3 standardise_validate.py --from-file ./tests/data/dem.json --preset dem_lerc --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01
+          cmp --silent ${HOME}/tmp/BK39_10000_0102.tiff ./scripts/tests/data/output/BK39_10000_0102.tiff
+          cmp --silent ${HOME}/tmp/BK39_10000_0101.tiff ./scripts/tests/data/output/BK39_10000_0101.tiff
+
+      - name: End to end test - Historical Aerial Imagery
+        run: |
+          # Standardising test 
+          docker run  -v ${HOME}/tmp/:/tmp/ topo-imagery python3 standardise_validate.py --from-file ./tests/data/hi.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01
+          cmp --silent ${HOME}/tmp/BQ31_5000_0608.tiff ./scripts/tests/data/output/BQ31_5000_0608.tiff
+
+      - name: End to end test - Cutline (Aerial Imagery)
+        run: |
+          # Standardising test 
+          docker run  -v ${HOME}/tmp/:/tmp/ topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/cutline/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --cutline ./tests/data/cutline_aerial.fgb
+          cmp --silent ${HOME}/tmp/cutline/BG35_1000_4829.tiff ./scripts/tests/data/output/BG35_1000_4829_cut.tiff
+
+      - name: End to end test - ASCII (Elevation)
+        run: |
+          # Standardising test 
+          docker run  -v ${HOME}/tmp/:/tmp/ topo-imagery python3 translate_ascii.py --from-file ./tests/data/elevation_ascii.json --target /tmp/translated/
+          cmp --silent ${HOME}/tmp/translated/elevation_ascii.tiff ./scripts/tests/data/output/elevation_ascii.tiff
diff --git a/scripts/collection_from_items.py b/scripts/collection_from_items.py
@@ -7,7 +7,7 @@
 from linz_logger import get_log
 
 from scripts.cli.cli_helper import coalesce_multi_single
-from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_json_in_uri
+from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_uri
 from scripts.logging.time_helper import time_in_ms
 from scripts.stac.imagery.collection import ImageryCollection
 from scripts.stac.imagery.provider import Provider, ProviderRole
@@ -54,7 +54,7 @@ def main() -> None:
 
     s3_client = client("s3")
 
-    files_to_read = list_json_in_uri(uri, s3_client)
+    files_to_read = list_uri(uri, s3_client)
 
     start_time = time_in_ms()
     for key, result in get_object_parallel_multithreading(

diff --git a/scripts/files/fs_s3.py b/scripts/files/fs_s3.py
@@ -164,7 +164,7 @@ def prefix_from_path(path: str) -> str:
     return path.replace(f"s3://{bucket_name}/", "")
 
 
-def list_json_in_uri(uri: str, s3_client: Optional[boto3.client]) -> List[str]:
+def list_uri(uri: str, s3_client: Optional[boto3.client]) -> List[str]:
     """Get the `JSON` files from a s3 path
 
     Args:
@@ -184,7 +184,7 @@ def list_json_in_uri(uri: str, s3_client: Optional[boto3.client]) -> List[str]:
         for contents_data in response["Contents"]:
             key = contents_data["Key"]
             if not is_json(key):
-                get_log().trace("skipping file not json", file=key, action="collection_from_items", reason="skip")
+                get_log().trace("skipping file not JSON", file=key, action="collection_from_items", reason="skip")
                 continue
             files.append(key)
     get_log().info("Files Listed", number_of_files=len(files))

diff --git a/scripts/gdal/gdal_preset.py b/scripts/gdal/gdal_preset.py
@@ -242,3 +242,23 @@ def get_thumbnail_command(
     command.extend(extra_args)
 
     return command
+
+
+def get_ascii_translate_command() -> List[str]:
+    """Get a `translate` command to transform the ascii files to tiff.
+
+    Args:
+
+    Returns:
+        a list of arguments to run `gdal_translate`
+    """
+    return [
+        "gdal_translate",
+        "-of",
+        "GTiff",
+        # Ensure all CPUs are used for gdal translate
+        "-co",
+        "num_threads=all_cpus",
+        "-co",
+        "COMPRESS=lzw",
+    ]
diff --git a/scripts/tests/data/elevation_ascii.asc b/scripts/tests/data/elevation_ascii.asc
@@ -0,0 +1,35 @@
+ncols        19
+nrows        29
+xllcorner    1579360.000000000000
+yllcorner    5219515.000000000000
+cellsize     25.000000000000
+NODATA_value  -9999
+ 26.95999908447265625 28.059999465942382812 28.780000686645507812 27.909999847412109375 22.649999618530273438 22.979999542236328125 28.1100006103515625 28.350000381469726562 28.149999618530273438 28.090000152587890625 27.93000030517578125 27.760000228881835938 27.559999465942382812 27.3600006103515625 27.049999237060546875 26.93000030517578125 26.68000030517578125 26.45999908447265625 26.3899993896484375
+ 28.809999465942382812 28.68000030517578125 26.70999908447265625 28.329999923706054688 22.350000381469726562 22.25 26.520000457763671875 28.270000457763671875 28.159999847412109375 28.020000457763671875 27.93000030517578125 27.70999908447265625 27.479999542236328125 27.29000091552734375 27.149999618530273438 26.920000076293945312 26.6100006103515625 26.45999908447265625 26.350000381469726562
+ 28.8600006103515625 28.770000457763671875 28.340000152587890625 22.729999542236328125 22.090000152587890625 22.450000762939453125 25.54000091552734375 28.030000686645507812 28.129999160766601562 27.870000839233398438 27.870000839233398438 27.68000030517578125 27.510000228881835938 27.299999237060546875 27.329999923706054688 26.93000030517578125 26.690000534057617188 26.479999542236328125 26.299999237060546875
+ 28.829999923706054688 28.729999542236328125 28.56999969482421875 23.340000152587890625 24.93000030517578125 22.059999465942382812 21.3899993896484375 26.809999465942382812 27.969999313354492188 27.829999923706054688 27.729999542236328125 27.700000762939453125 27.510000228881835938 27.229999542236328125 27.1100006103515625 26.8899993896484375 26.659999847412109375 26.489999771118164062 26.20999908447265625
+ 28.739999771118164062 28.56999969482421875 28.549999237060546875 28.399999618530273438 28.29000091552734375 24.31999969482421875 21.590000152587890625 22.149999618530273438 25.020000457763671875 27.770000457763671875 27.81999969482421875 27.6399993896484375 27.489999771118164062 27.239999771118164062 27.020000457763671875 26.969999313354492188 26.6100006103515625 26.479999542236328125 26.3899993896484375
+ 28.70999908447265625 28.56999969482421875 28.3899993896484375 28.3899993896484375 28.219999313354492188 28.030000686645507812 22.93000030517578125 20.8899993896484375 21.450000762939453125 22.3600006103515625 27.549999237060546875 27.600000381469726562 27.399999618530273438 27.090000152587890625 27.059999465942382812 26.8899993896484375 26.739999771118164062 26.629999160766601562 26.3899993896484375
+ 28.6399993896484375 28.590000152587890625 28.3899993896484375 28.190000534057617188 28.219999313354492188 28.010000228881835938 21.45999908447265625 21.229999542236328125 21.1399993896484375 20.700000762939453125 20.479999542236328125 27.549999237060546875 24.229999542236328125 26.340000152587890625 27.020000457763671875 26.81999969482421875 26.729999542236328125 26.739999771118164062 26.379999160766601562
+ 28.620000839233398438 28.379999160766601562 28.340000152587890625 28.190000534057617188 28.059999465942382812 26.8600006103515625 21.090000152587890625 23.729999542236328125 20.850000381469726562 21.030000686645507812 20.549999237060546875 19.909999847412109375 20.149999618530273438 19.75 20.969999313354492188 19.340000152587890625 19.56999969482421875 24.690000534057617188 26.379999160766601562
+ 28.549999237060546875 28.43000030517578125 28.379999160766601562 28.200000762939453125 28.079999923706054688 28 23.159999847412109375 21.530000686645507812 25.159999847412109375 23.20999908447265625 20.31999969482421875 20.270000457763671875 19.700000762939453125 19.450000762939453125 19.520000457763671875 19.120000839233398438 21.68000030517578125 23.049999237060546875 26.45999908447265625
+ 28.549999237060546875 28.3600006103515625 28.229999542236328125 28.25 28.04000091552734375 27.950000762939453125 27.799999237060546875 27.700000762939453125 27.450000762939453125 27.299999237060546875 20.909999847412109375 19.6100006103515625 20.010000228881835938 19.409999847412109375 19.120000839233398438 18.770000457763671875 18.25 27.020000457763671875 26.379999160766601562
+ 28.6399993896484375 28.5 28.370000839233398438 28.370000839233398438 28.129999160766601562 28.010000228881835938 27.739999771118164062 27.489999771118164062 27.31999969482421875 27.25 26.299999237060546875 20.270000457763671875 19.8600006103515625 19.25 18.760000228881835938 25.739999771118164062 18.6399993896484375 21.29000091552734375 26.68000030517578125
+ 28.649999618530273438 28.719999313354492188 28.649999618530273438 28.3899993896484375 28.159999847412109375 27.950000762939453125 27.81999969482421875 27.530000686645507812 27.440000534057617188 27.149999618530273438 27.079999923706054688 26.879999160766601562 21.219999313354492188 18.729999542236328125 19.030000686645507812 20.620000839233398438 18.6399993896484375 18.659999847412109375 26.45999908447265625
+ 28.840000152587890625 28.8600006103515625 28.68000030517578125 28.379999160766601562 28.129999160766601562 27.899999618530273438 27.8600006103515625 27.629999160766601562 27.370000839233398438 27.229999542236328125 26.909999847412109375 26.93000030517578125 26.950000762939453125 23.260000228881835938 18.6100006103515625 18.270000457763671875 18.440000534057617188 18.06999969482421875 26.719999313354492188
+ 29.100000381469726562 29 28.579999923706054688 28.280000686645507812 28.100000381469726562 27.8899993896484375 27.760000228881835938 27.629999160766601562 27.440000534057617188 27.1399993896484375 27.04000091552734375 26.780000686645507812 26.739999771118164062 26.620000839233398438 18.75 18.020000457763671875 17.920000076293945312 17.70999908447265625 26.95999908447265625
+ 29.299999237060546875 28.979999542236328125 28.6399993896484375 28.379999160766601562 28.20999908447265625 27.979999542236328125 27.809999465942382812 27.690000534057617188 27.620000839233398438 27.31999969482421875 26.95999908447265625 26.969999313354492188 23.670000076293945312 22.870000839233398438 19.1100006103515625 18.059999465942382812 16.95999908447265625 18.729999542236328125 24.700000762939453125
+ 29.340000152587890625 29.090000152587890625 28.809999465942382812 28.549999237060546875 28.229999542236328125 27.969999313354492188 27.840000152587890625 27.440000534057617188 27.43000030517578125 27.049999237060546875 27.090000152587890625 26.309999465942382812 23.350000381469726562 20.81999969482421875 17.620000839233398438 17.68000030517578125 18.409999847412109375 17.469999313354492188 17.68000030517578125
+ 29.469999313354492188 29.25 28.979999542236328125 28.530000686645507812 28.079999923706054688 27.8600006103515625 27.670000076293945312 27.3899993896484375 27.1100006103515625 27.010000228881835938 24.829999923706054688 17.8600006103515625 16.93000030517578125 16.879999160766601562 17.329999923706054688 17.219999313354492188 17.620000839233398438 16.909999847412109375 25.6399993896484375
+ 29.450000762939453125 29.229999542236328125 28.920000076293945312 28.510000228881835938 28.049999237060546875 27.739999771118164062 27.489999771118164062 27.29000091552734375 27.1399993896484375 26.75 25.1399993896484375 16.5 16.190000534057617188 16.420000076293945312 18.799999237060546875 17.190000534057617188 17.629999160766601562 17.059999465942382812 26.1100006103515625
+ 29.260000228881835938 29.010000228881835938 28.739999771118164062 28.399999618530273438 28.1100006103515625 27.6399993896484375 27.520000457763671875 27.270000457763671875 26.95999908447265625 26.559999465942382812 16.70999908447265625 16.329999923706054688 16.280000686645507812 16.8600006103515625 18.559999465942382812 17.940000534057617188 16.54000091552734375 16.590000152587890625 25.840000152587890625
+ 29 28.770000457763671875 28.530000686645507812 28.170000076293945312 28.159999847412109375 27.909999847412109375 27.420000076293945312 27.170000076293945312 26.879999160766601562 26.469999313354492188 17.479999542236328125 16.45999908447265625 15.81999969482421875 16.45999908447265625 16.280000686645507812 15.56999969482421875 16.479999542236328125 16.600000381469726562 25.729999542236328125
+ 28.8600006103515625 28.530000686645507812 28.43000030517578125 28.25 27.81999969482421875 27.75 27.409999847412109375 27.04000091552734375 26.829999923706054688 26.56999969482421875 26.239999771118164062 21.899999618530273438 16.309999465942382812 15.760000228881835938 16.129999160766601562 16.549999237060546875 15.710000038146972656 20.579999923706054688 25.56999969482421875
+ 28.729999542236328125 28.549999237060546875 28.010000228881835938 27.850000381469726562 27.68000030517578125 27.600000381469726562 27.3899993896484375 26.969999313354492188 26.79000091552734375 26.590000152587890625 26.260000228881835938 25.95999908447265625 18.280000686645507812 15.420000076293945312 16.1399993896484375 15.159999847412109375 15.56999969482421875 17.479999542236328125 25.600000381469726562
+ 28.440000534057617188 28.239999771118164062 27.8899993896484375 27.620000839233398438 27.56999969482421875 27.43000030517578125 27.1100006103515625 26.879999160766601562 26.739999771118164062 26.489999771118164062 26.219999313354492188 26.020000457763671875 17.25 16.010000228881835938 16.030000686645507812 15.729999542236328125 15.5 15.310000419616699219 17.809999465942382812
+ 28.280000686645507812 28.010000228881835938 27.719999313354492188 27.520000457763671875 27.299999237060546875 27.129999160766601562 26.969999313354492188 27.059999465942382812 26.620000839233398438 26.489999771118164062 26.399999618530273438 24.659999847412109375 15.090000152587890625 14.909999847412109375 14.760000228881835938 14.760000228881835938 15.449999809265136719 14.949999809265136719 15.850000381469726562
+ 28.260000228881835938 27.95999908447265625 27.54000091552734375 27.219999313354492188 26.979999542236328125 26.8899993896484375 26.79000091552734375 26.70999908447265625 26.420000076293945312 26.340000152587890625 26.29000091552734375 24.100000381469726562 15.659999847412109375 21.309999465942382812 19.239999771118164062 16.299999237060546875 14.789999961853027344 14.659999847412109375 14.710000038146972656
+ 28.200000762939453125 27.8600006103515625 27.479999542236328125 27.170000076293945312 26.8600006103515625 26.700000762939453125 26.56999969482421875 26.370000839233398438 26.379999160766601562 26.1100006103515625 26.020000457763671875 25.940000534057617188 25.6399993896484375 25.510000228881835938 25.280000686645507812 19.1399993896484375 14.859999656677246094 14.260000228881835938 13.829999923706054688
+ 28.030000686645507812 27.729999542236328125 27.530000686645507812 27.149999618530273438 26.760000228881835938 26.440000534057617188 26.3600006103515625 26.219999313354492188 26.399999618530273438 26 25.8600006103515625 25.719999313354492188 25.579999923706054688 25.440000534057617188 25.190000534057617188 24.3600006103515625 16.25 14.060000419616699219 14.720000267028808594
+ 27.8600006103515625 27.719999313354492188 27.379999160766601562 27.010000228881835938 26.549999237060546875 26.25 26.059999465942382812 26.06999969482421875 26.030000686645507812 25.840000152587890625 25.68000030517578125 25.510000228881835938 25.3899993896484375 25.260000228881835938 25.1399993896484375 25.120000839233398438 24.799999237060546875 16.909999847412109375 14.189999580383300781
+ 27.489999771118164062 27.280000686645507812 27.100000381469726562 26.950000762939453125 26.54000091552734375 26.06999969482421875 25.979999542236328125 25.75 25.559999465942382812 25.659999847412109375 25.469999313354492188 25.370000839233398438 25.270000457763671875 25.1100006103515625 24.95999908447265625 25.159999847412109375 25.260000228881835938 24.780000686645507812 17.409999847412109375
diff --git a/scripts/tests/data/elevation_ascii.json b/scripts/tests/data/elevation_ascii.json
@@ -0,0 +1 @@
+["./tests/data/elevation_ascii.asc"]
diff --git a/scripts/tests/data/output/elevation_ascii.tiff b/scripts/tests/data/output/elevation_ascii.tiff
diff --git a/scripts/translate_ascii.py b/scripts/translate_ascii.py
@@ -0,0 +1,46 @@
+import argparse
+import json
+import os
+import tempfile
+
+from scripts.files.fs import find_sidecars, read, write_all
+from scripts.gdal.gdal_helper import run_gdal
+from scripts.gdal.gdal_preset import get_ascii_translate_command
+
+
+def main() -> None:
+    """Translates ascii files in a given path to tiffs and writes to target,
+
+    Arguments:
+    --from-file - file listing the source data
+    --target - local or s3 path to write converted tiffs
+
+    examples:
+        python translate_ascii.py --from-file s3://linz-elevation-staging/test/sample-tests/file-list.json --target /tmp/
+        python translate_ascii.py --from-file ./tests/data/file-list.json --target /tmp/
+    """
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--from-file", dest="from_file", required=True, help="Path to file listing ascii files")
+    parser.add_argument("--target", dest="target", required=True, help="Output location path")
+    arguments = parser.parse_args()
+
+    asc_files = json.loads(read(arguments.from_file))
+
+    with tempfile.TemporaryDirectory() as tmp_path:
+        tiffs = []
+        for asc in asc_files:
+            # translate from ascii to geotiff using GDAL
+            filename = os.path.splitext(os.path.basename(asc))[0]
+            tiff = os.path.join(tmp_path, f"{filename}.tiff")
+            run_gdal(get_ascii_translate_command(), input_file=asc, output_file=tiff)
+            tiffs.append(tiff)
+        write_all(inputs=tiffs, target=arguments.target)
+
+        # copy any sidecar files to target
+        sidecars = find_sidecars(asc_files, [".prj", ".tfw"])
+        write_all(inputs=sidecars, target=arguments.target)
+
+
+if __name__ == "__main__":
+    main()