From 47c2892495bcab61926b28115af8bce5e562ab55 Mon Sep 17 00:00:00 2001
From: Megan Davidson <mdavidson@linz.govt.nz>
Date: Mon, 28 Aug 2023 13:15:09 +1200
Subject: [PATCH 1/3] feat: ascii standardise publish workflow

---
 .../imagery/ascii-standardise-publish.yaml    | 172 ++++++++++++++++++
 workflows/imagery/standardising.yaml          |   5 +-
 2 files changed, 176 insertions(+), 1 deletion(-)
 create mode 100644 workflows/imagery/ascii-standardise-publish.yaml

diff --git a/workflows/imagery/ascii-standardise-publish.yaml b/workflows/imagery/ascii-standardise-publish.yaml
new file mode 100644
index 00000000..3f958420
--- /dev/null
+++ b/workflows/imagery/ascii-standardise-publish.yaml
@@ -0,0 +1,172 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: ascii-imagery-standardising-publish-
+  namespace: argo
+spec:
+  parallelism: 50
+  nodeSelector:
+    karpenter.sh/capacity-type: "spot"
+  entrypoint: main
+  synchronization:
+    semaphore:
+      configMapKeyRef:
+        name: semaphores
+        key: bulk
+  arguments:
+    parameters:
+      - name: cutline # optional standardising cutline
+        value: ""
+      - name: compression
+        value: "webp"
+      - name: source-epsg
+        value: "2193"
+      - name: target-epsg
+        value: "2193"
+      - name: group
+        value: "50"
+      - name: copy-option
+        value: "--no-clobber"
+      - name: include
+        value: ".tiff?$"
+      - name: transform
+        value: "f"
+      - name: validate
+        value: "true"
+      - name: retile
+        value: "false"
+      - name: version-argo-tasks
+        value: "v2"
+      - name: version-basemaps-cli
+        value: "v6"
+      - name: version-topo-imagery
+        value: "v3"
+  templateDefaults:
+    container:
+      imagePullPolicy: Always
+  templates:
+    - name: main
+      dag:
+        tasks:
+          - name: get-location
+            template: get-location
+
+          - name: aws-list
+            template: aws-list
+
+          - name: convert-asc-files
+            template: convert-asc-files
+            arguments:
+              parameters:
+                - name: target
+                  value: "{{tasks.get-location.outputs.parameters.location}}translated-asc-files/"
+              artifacts:
+                - name: files
+                  from: "{{ tasks.aws-list.outputs.artifacts.files }}"
+            depends: "get-location && aws-list"
+
+          - name: standardise
+            templateRef:
+              name: imagery-standardising
+              template: main
+            arguments:
+              parameters:
+                - name: source
+                  value: "{{tasks.get-location.outputs.parameters.location}}translated-asc-files/"
+            depends: "convert-asc-files"
+
+          - name: publish
+            templateRef:
+              name: publish-copy
+              template: main
+            arguments:
+              parameters:
+                - name: source
+                  value: "{{tasks.get-location.outputs.parameters.location}}flat/"
+                - name: include
+                  value: ".tiff?$|.json$"
+                - name: group
+                  value: "1000"
+                - name: group-size
+                  value: "100Gi"
+            depends: "standardise"
+        # END TEMPLATE `main`
+
+    - name: get-location
+      script:
+        image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}"
+        command: [node]
+        source: |
+          const fs = require('fs');
+          const loc = JSON.parse(process.env['ARGO_TEMPLATE']).archiveLocation.s3;
+          const key = loc.key.replace('{{pod.name}}','');
+          fs.writeFileSync('/tmp/location', `s3://${loc.bucket}/${key}`);
+      outputs:
+        parameters:
+          - name: location
+            valueFrom:
+              path: "/tmp/location"
+
+    - name: aws-list
+      inputs:
+      container:
+        image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{workflow.parameters.version-argo-tasks}}
+        command: [node, /app/index.js]
+        env:
+          - name: AWS_ROLE_CONFIG_PATH
+            value: s3://linz-bucket-config/config.json
+        args:
+          [
+            "list",
+            "--verbose",
+            "--include",
+            ".asc?$",
+            "--group",
+            "999999",
+            "--output",
+            "/tmp/file_list.json",
+            "{{workflow.parameters.source}}",
+          ]
+      outputs:
+        artifacts:
+          - name: files
+            path: /tmp/file_list.json
+
+    - name: convert-asc-files
+      retryStrategy:
+        limit: "2"
+      nodeSelector:
+        karpenter.sh/capacity-type: "spot"
+      inputs:
+        parameters:
+          - name: target
+        artifacts:
+          - name: files
+            path: /tmp/file_list.json
+      container:
+        image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}"
+        resources:
+          requests:
+            memory: 7.8Gi
+            cpu: 15000m
+            ephemeral-storage: 3Gi
+        volumeMounts:
+          - name: ephemeral
+            mountPath: "/tmp"
+        command:
+          - python
+          - "/app/scripts/translate_ascii.py"
+        args:
+          - "--from-file"
+          - "/tmp/file_list.json"
+          - "--target"
+          - "{{inputs.parameters.target}}"
+
+  volumes:
+    - name: ephemeral
+      emptyDir: {}
+    - name: secret-volume # this is for the publish-copy step
+      secret:
+        secretName: github-linz-imagery
+        defaultMode: 384
diff --git a/workflows/imagery/standardising.yaml b/workflows/imagery/standardising.yaml
index 020a5138..a8aad7f0 100644
--- a/workflows/imagery/standardising.yaml
+++ b/workflows/imagery/standardising.yaml
@@ -212,6 +212,9 @@ spec:
       imagePullPolicy: Always
   templates:
     - name: main
+      inputs:
+        parameters:
+          - name: source
       dag:
         tasks:
           - name: collection-id-setup
@@ -228,7 +231,7 @@ spec:
                 - name: include
                   value: "{{workflow.parameters.include}}"
                 - name: source
-                  value: "{{=sprig.trim(workflow.parameters.source)}}"
+                  value: "{{=sprig.trim(inputs.parameters.source)}}"
                 - name: source_epsg
                   value: "{{=sprig.trim(workflow.parameters['source-epsg'])}}"
                 - name: validate

From 1df9ded7e8e99874ba407ba162b191af2a07e0e5 Mon Sep 17 00:00:00 2001
From: Megan Davidson <mdavidson@linz.govt.nz>
Date: Mon, 28 Aug 2023 13:26:17 +1200
Subject: [PATCH 2/3] fix: aws-list requires a source path

---
 workflows/imagery/ascii-standardise-publish.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/workflows/imagery/ascii-standardise-publish.yaml b/workflows/imagery/ascii-standardise-publish.yaml
index 3f958420..4dd7d1d8 100644
--- a/workflows/imagery/ascii-standardise-publish.yaml
+++ b/workflows/imagery/ascii-standardise-publish.yaml
@@ -16,6 +16,8 @@ spec:
         key: bulk
   arguments:
     parameters:
+      - name: source
+        value: "s3://path/"
       - name: cutline # optional standardising cutline
         value: ""
       - name: compression

From 98c4e65541c00161a786e5f69ee8f2a0387a33da Mon Sep 17 00:00:00 2001
From: Megan Davidson <mdavidson@linz.govt.nz>
Date: Wed, 6 Sep 2023 15:15:54 +1200
Subject: [PATCH 3/3] fix: update parameters

and fix minor formatting issue
---
 workflows/imagery/ascii-standardise-publish.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/workflows/imagery/ascii-standardise-publish.yaml b/workflows/imagery/ascii-standardise-publish.yaml
index 4dd7d1d8..ebef13f8 100644
--- a/workflows/imagery/ascii-standardise-publish.yaml
+++ b/workflows/imagery/ascii-standardise-publish.yaml
@@ -20,6 +20,8 @@ spec:
         value: "s3://path/"
       - name: cutline # optional standardising cutline
         value: ""
+      - name: collection-id # optional
+        value: ""
       - name: compression
         value: "webp"
       - name: source-epsg
@@ -113,7 +115,7 @@ spec:
     - name: aws-list
       inputs:
       container:
-        image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{workflow.parameters.version-argo-tasks}}
+        image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{workflow.parameters.version-argo-tasks}}"
         command: [node, /app/index.js]
         env:
           - name: AWS_ROLE_CONFIG_PATH