fix: get-location should output the s3 archive workflow directory TDE…

…-1280

templates/common/README.md

@@ -89,3 +89,8 @@ spec:
                 - name: workflow_parameters
                   value: '{{workflow.parameters}}'
 ```
+
+## Get Location - `tpl-get-location`
+
+Template to output the S3 Archive location for the workflow (example: `s3://linz-workflows-scratch/2024-10/02-my-workflow-29l4x/`).
+In some cases, we need this location to write output files by the workflow in a specific and consistent "folder" within the archive bucket.

templates/common/__test__/exit.handler.test.ts

@@ -1,37 +1,22 @@
 import assert from 'node:assert';
-import fs from 'node:fs';
 import { describe, it } from 'node:test';
 
-/**
- * Read the workflow YAML file and create a function from the script inside.
- * replacing {{ inputs.* }} with ctx
- *
- * @param ctx
- */
-function runTestFunction(ctx: { workflowParameters: string; workflowStatus: string }): void {
-  const func = fs.readFileSync('./templates/common/exit.handler.yml', 'utf-8').split('source: |')[1];
-  if (!func) {
-    throw new Error('No script found in the workflow');
-  }
-  const newFunc = func
-    // Replace inputs
-    .replace('{{= inputs.parameters.workflow_parameters }}', `${ctx.workflowParameters ?? '[]'}`)
-    .replace('{{inputs.parameters.workflow_status}}', `${ctx.workflowStatus ?? 'Failed'}`);
-  // eslint-disable-next-line @typescript-eslint/no-implied-eval
-  new Function(newFunc)();
-}
+import { runTestFunction } from './function.helper.js';
 
 describe('exit handler script template', () => {
   it('should log workflow status and parameters', (t) => {
     const spy = t.mock.method(console, 'log');
 
-    runTestFunction({
-      workflowParameters: JSON.stringify([
-        { name: 'source', value: 's3://linz-topographic-upload/abc/', description: 'Source bucket' },
-        { name: 'ticket', value: 'GDE-123', description: 'JIRA Ticket' },
-      ]),
-      workflowStatus: `Succeeded`,
-    });
+    runTestFunction('./templates/common/exit.handler.yml', [
+      {
+        toReplace: '{{= inputs.parameters.workflow_parameters }}',
+        replaceWith: JSON.stringify([
+          { name: 'source', value: 's3://linz-topographic-upload/abc/', description: 'Source bucket' },
+          { name: 'ticket', value: 'GDE-123', description: 'JIRA Ticket' },
+        ]),
+      },
+      { toReplace: '{{inputs.parameters.workflow_status}}', replaceWith: 'Succeeded' },
+    ]);
 
     assert.equal(spy.mock.callCount(), 1);
 

templates/common/__test__/function.helper.ts

@@ -0,0 +1,40 @@
+import fs from 'node:fs';
+
+/**
+ * Extract the script from a workflow template file located as its `source`.
+ *
+ * @param path to the workflow template file
+ * @param stopDelimiter optional delimiter to stop the extraction
+ * @returns the script as a string
+ */
+function getFunctionFromScript(path: string, stopDelimiter?: string): string {
+  let func = fs.readFileSync(path, 'utf-8').split('source: |')[1];
+  if (stopDelimiter) {
+    func = func?.split(stopDelimiter)[0];
+  }
+  if (!func) {
+    throw new Error('No script found in the workflow');
+  }
+  return func;
+}
+
+/**
+ * Data to replace in the script
+ */
+type FunctionData = { toReplace: string; replaceWith: string };
+
+/**
+ * Create and run a function from a script located in a workflowTemplate file.
+ *
+ * @param workflowPath path to the workflow template file
+ * @param data data to replace in the script
+ * @param stopDelimiter optional delimiter to stop the extraction of the script
+ */
+export function runTestFunction(workflowPath: string, data: FunctionData[], stopDelimiter?: string): void {
+  let func = getFunctionFromScript(workflowPath, stopDelimiter);
+  data.forEach((d) => {
+    func = func.replace(d.toReplace, d.replaceWith);
+  });
+  // eslint-disable-next-line @typescript-eslint/no-implied-eval
+  new Function(func)();
+}

templates/common/__test__/get.location.test.ts

@@ -0,0 +1,41 @@
+import assert from 'node:assert';
+import { describe, it } from 'node:test';
+
+import { runTestFunction } from './function.helper.js';
+
+describe('get-location script template', () => {
+  it('should output workflow artifact location', (t) => {
+    const spy = t.mock.method(console, 'log');
+    runTestFunction(
+      './templates/common/get.location.yml',
+      [
+        {
+          toReplace: '/tmp/',
+          replaceWith: 'memory://tmp/',
+        },
+        {
+          toReplace: `JSON.parse(process.env['ARGO_TEMPLATE'])`,
+          replaceWith: JSON.stringify({
+            archiveLocation: {
+              s3: {
+                key: '2024-10/02-test-get-location-29l4x/test-get-location-29l4x-get-location-3125883809',
+                bucket: 'linz-workflows-scratch',
+              },
+            },
+          }),
+        },
+      ],
+      '// Write outputs',
+    );
+    assert.equal(spy.mock.callCount(), 3);
+
+    const location = String(spy.mock.calls[0]?.arguments[0]);
+    assert.equal(location, 'Location: s3://linz-workflows-scratch/2024-10/02-test-get-location-29l4x/');
+
+    const bucket = String(spy.mock.calls[1]?.arguments[0]);
+    assert.equal(bucket, 'Bucket: linz-workflows-scratch');
+
+    const key = String(spy.mock.calls[2]?.arguments[0]);
+    assert.equal(key, 'Key: 2024-10/02-test-get-location-29l4x');
+  });
+});

templates/common/get.location.yml

@@ -0,0 +1,47 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json
+
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  # Template to output the locaton of the workflow S3 artifact root directory.
+  name: tpl-get-location
+spec:
+  templateDefaults:
+    container:
+      imagePullPolicy: Always
+      image: ''
+  entrypoint: main
+  templates:
+    - name: main
+      inputs:
+        parameters:
+          - name: version_argo_tasks
+            description: Version of argo-tasks to use
+            default: 'v4'
+      outputs:
+        parameters:
+          - name: location
+            valueFrom:
+              path: '/tmp/location'
+          - name: bucket
+            valueFrom:
+              path: '/tmp/bucket'
+          - name: key
+            valueFrom:
+              path: '/tmp/key'
+      script:
+        image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(inputs.parameters.version_argo_tasks)}}'
+        command: [node]
+        source: |
+          const argoTemplate = JSON.parse(process.env['ARGO_TEMPLATE']);
+          const podArchiveLoc = argoTemplate.archiveLocation.s3;
+          const workflowArchiveKey = podArchiveLoc.key.substr(0, podArchiveLoc.key.lastIndexOf('/'));
+          const location = `s3://${podArchiveLoc.bucket}/${workflowArchiveKey}/`
+          console.log(`Location: ${location}`);
+          console.log(`Bucket: ${podArchiveLoc.bucket}`);
+          console.log(`Key: ${workflowArchiveKey}`);
+          // Write outputs
+          const fs = require('fs');
+          fs.writeFileSync('/tmp/location', location);
+          fs.writeFileSync('/tmp/bucket', `${podArchiveLoc.bucket}`);
+          fs.writeFileSync('/tmp/key', `${workflowArchiveKey}`);

workflows/raster/standardising.yaml

@@ -426,7 +426,9 @@ spec:
             depends: 'create-collection'
 
           - name: get-location
-            template: get-location
+            templateRef:
+              name: tpl-get-location
+              template: main
 
           - name: create-overview
             when: "'{{workflow.parameters.target_epsg}}' =~ '2193|3857' && '{{workflow.parameters.compression}}' != 'dem_lerc'"
@@ -591,29 +593,6 @@ spec:
           - '--concurrency'
           - '25'
 
-    - name: get-location
-      script:
-        image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}'
-        command: [node]
-        source: |
-          const fs = require('fs');
-          const loc = JSON.parse(process.env['ARGO_TEMPLATE']).archiveLocation.s3;
-          const key = loc.key.replace('{{pod.name}}','');
-          fs.writeFileSync('/tmp/location', `s3://${loc.bucket}/${key}`);
-          fs.writeFileSync('/tmp/bucket', `${loc.bucket}`);
-          fs.writeFileSync('/tmp/key', `${key}`);
-      outputs:
-        parameters:
-          - name: location
-            valueFrom:
-              path: '/tmp/location'
-          - name: bucket
-            valueFrom:
-              path: '/tmp/bucket'
-          - name: key
-            valueFrom:
-              path: '/tmp/key'
-
     - name: create-overview
       inputs:
         parameters:

workflows/util/create-thumbnails.yaml

@@ -35,7 +35,9 @@ spec:
             template: aws-list
 
           - name: get-location
-            template: get-location
+            templateRef:
+              name: tpl-get-location
+              template: main
 
           - name: thumbnails
             template: thumbnails
@@ -100,21 +102,6 @@ spec:
           - '--target'
           - '{{inputs.parameters.target}}'
 
-    - name: get-location
-      script:
-        image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}'
-        command: [node]
-        source: |
-          const fs = require('fs');
-          const loc = JSON.parse(process.env['ARGO_TEMPLATE']).archiveLocation.s3;
-          const key = loc.key.replace('{{pod.name}}','');
-          fs.writeFileSync('/tmp/location', `s3://${loc.bucket}/${key}`);
-      outputs:
-        parameters:
-          - name: location
-            valueFrom:
-              path: '/tmp/location'
-
   volumes:
     - name: ephemeral
       emptyDir: {}