Merge branch 'master' into feat/exit-handler-log-workflowgroup-tde-1230

templates/argo-tasks/README.md

@@ -136,6 +136,8 @@ Create a manifest file for a user specified source and target that includes `.ti
         value: '\.tiff?$|\.json$|\.tfw$'
       - name: exclude
         value: ''
+      - name: flatten
+        value: 'false'
       - name: group
         value: '1000'
       - name: group_size

templates/argo-tasks/create-manifest.yml

@@ -42,6 +42,13 @@ spec:
               - '--force'
               - '--force-no-clobber'
 
+          - name: flatten
+            description: Flatten the files in the target location
+            default: 'false'
+            enum:
+              - 'true'
+              - 'false'
+
           - name: group
             description: The maximum number of files for each pod to copy (will use the value of `group` or `group_size` that is reached first)
             default: '1000'
@@ -74,6 +81,7 @@ spec:
             '{{=sprig.trim(inputs.parameters.include)}}',
             '--exclude',
             '{{inputs.parameters.exclude}}',
+            '--flatten={{=sprig.trim(inputs.parameters.flatten)}}',
             '--group',
             '{{=sprig.trim(inputs.parameters.group)}}',
             '--group-size',

templates/common/exit.handler.yml

@@ -48,7 +48,8 @@ spec:
               time: Date.now(),
               level: 20,
               pid: 1,
-              msg: 'Workflow:{{inputs.parameters.workflow_status}}',
+              msg: 'Workflow:Done',
+              workflowStatus: '{{inputs.parameters.workflow_status}}', 
               workflowGroup: guessWorkflowGroup(),
               parameters,
             }),

workflows/raster/README.md

@@ -210,11 +210,12 @@ Access permissions are controlled by the [Bucket Sharing Config](https://github.
 | -------------------- | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | ticket               | str   |                                                                                                                                                               | Ticket ID e.g. 'AIP-55'                                                                                                                                                                                                     |
 | region               | enum  |                                                                                                                                                               | Region of the dataset                                                                                                                                                                                                       |
-| source               | str   | s3://linz-imagery-staging/test/sample/                                                                                                                        | The URIs (paths) to the s3 source location                                                                                                                                                                                  |
+| source               | str   | s3://linz-imagery-staging/test/sample/                                                                                                                        | The URIs (paths) to the s3 source location. Separate multiple source paths with `;`                                                                                                                                         |
 | target               | str   | s3://linz-imagery-staging/test/sample_target/                                                                                                                 | The URIs (paths) to the s3 target location                                                                                                                                                                                  |
 | include              | regex | \\.tiff?\$\|\\.json\$\|\\.tfw$\|/capture-area\\.geojson$                                                                                                      | A regular expression to match object path(s) or name(s) from within the source path to include in the copy.                                                                                                                 |
 | exclude              | regex |                                                                                                                                                               | A regular expression to match object path(s) or name(s) from within the source path to exclude from the copy.                                                                                                                 |
 | copy_option          | enum  | --no-clobber                                                                                                                                                  | <dl><dt>`--no-clobber` </dt><dd> Skip overwriting existing files.</dd><dt> `--force` </dt><dd> Overwrite all files. </dd><dt> `--force-no-clobber` </dt><dd> Overwrite only changed files, skip unchanged files. </dd></dl> |
+| flatten              | enum  | false                                                                                                                                                         | Flatten the files in the target location (useful for multiple source locations)                                                                                                                                       |
 | group                | int   | 1000                                                                                                                                                          | The maximum number of files for each pod to copy (will use the value of `group` or `group_size` that is reached first).                                                                                                     |
 | group_size           | str   | 100Gi                                                                                                                                                         | The maximum group size of files for each pod to copy (will use the value of `group` or `group_size` that is reached first).                                                                                                 |
 | transform            | str   | `f`                                                                                                                                                           | String to be transformed from source to target to renamed filenames, e.g. `f.replace("text to replace", "new_text_to_use")`. Leave as `f` for no transformation.                                                            |

workflows/raster/copy.yaml

@@ -68,6 +68,12 @@ spec:
           - '--no-clobber'
           - '--force'
           - '--force-no-clobber'
+      - name: flatten
+        description: Flatten the files in the target location
+        default: 'false'
+        enum:
+          - 'true'
+          - 'false'
       - name: group
         value: '1000'
       - name: group_size
@@ -92,6 +98,7 @@ spec:
           - name: target
           - name: include
           - name: exclude
+          - name: flatten
           - name: group
           - name: group_size
       dag:
@@ -110,6 +117,8 @@ spec:
                   value: '{{inputs.parameters.include}}'
                 - name: exclude
                   value: '{{inputs.parameters.exclude}}'
+                - name: flatten
+                  value: '{{inputs.parameters.flatten}}'
                 - name: group
                   value: '{{inputs.parameters.group}}'
                 - name: group_size