diff --git a/templates/argo-tasks/README.md b/templates/argo-tasks/README.md index 063f16dd..27234f3d 100644 --- a/templates/argo-tasks/README.md +++ b/templates/argo-tasks/README.md @@ -136,6 +136,8 @@ Create a manifest file for a user specified source and target that includes `.ti value: '\.tiff?$|\.json$|\.tfw$' - name: exclude value: '' + - name: flatten + value: 'false' - name: group value: '1000' - name: group_size diff --git a/templates/argo-tasks/create-manifest.yml b/templates/argo-tasks/create-manifest.yml index dfb1b046..e5117fb5 100644 --- a/templates/argo-tasks/create-manifest.yml +++ b/templates/argo-tasks/create-manifest.yml @@ -42,6 +42,13 @@ spec: - '--force' - '--force-no-clobber' + - name: flatten + description: Flatten the files in the target location + default: 'false' + enum: + - 'true' + - 'false' + - name: group description: The maximum number of files for each pod to copy (will use the value of `group` or `group_size` that is reached first) default: '1000' @@ -74,6 +81,7 @@ spec: '{{=sprig.trim(inputs.parameters.include)}}', '--exclude', '{{inputs.parameters.exclude}}', + '--flatten={{=sprig.trim(inputs.parameters.flatten)}}', '--group', '{{=sprig.trim(inputs.parameters.group)}}', '--group-size', diff --git a/templates/common/exit.handler.yml b/templates/common/exit.handler.yml index 2d42a643..56d12b40 100644 --- a/templates/common/exit.handler.yml +++ b/templates/common/exit.handler.yml @@ -48,7 +48,8 @@ spec: time: Date.now(), level: 20, pid: 1, - msg: 'Workflow:{{inputs.parameters.workflow_status}}', + msg: 'Workflow:Done', + workflowStatus: '{{inputs.parameters.workflow_status}}', workflowGroup: guessWorkflowGroup(), parameters, }), diff --git a/workflows/raster/README.md b/workflows/raster/README.md index 6eb8522e..550f210b 100644 --- a/workflows/raster/README.md +++ b/workflows/raster/README.md @@ -210,11 +210,12 @@ Access permissions are controlled by the [Bucket Sharing Config](https://github. | -------------------- | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | ticket | str | | Ticket ID e.g. 'AIP-55' | | region | enum | | Region of the dataset | -| source | str | s3://linz-imagery-staging/test/sample/ | The URIs (paths) to the s3 source location | +| source | str | s3://linz-imagery-staging/test/sample/ | The URIs (paths) to the s3 source location. Separate multiple source paths with `;` | | target | str | s3://linz-imagery-staging/test/sample_target/ | The URIs (paths) to the s3 target location | | include | regex | \\.tiff?\$\|\\.json\$\|\\.tfw$\|/capture-area\\.geojson$ | A regular expression to match object path(s) or name(s) from within the source path to include in the copy. | | exclude | regex | | A regular expression to match object path(s) or name(s) from within the source path to exclude from the copy. | | copy_option | enum | --no-clobber |
`--no-clobber`
Skip overwriting existing files.
`--force`
Overwrite all files.
`--force-no-clobber`
Overwrite only changed files, skip unchanged files.
| +| flatten | enum | false | Flatten the files in the target location (useful for multiple source locations) | | group | int | 1000 | The maximum number of files for each pod to copy (will use the value of `group` or `group_size` that is reached first). | | group_size | str | 100Gi | The maximum group size of files for each pod to copy (will use the value of `group` or `group_size` that is reached first). | | transform | str | `f` | String to be transformed from source to target to renamed filenames, e.g. `f.replace("text to replace", "new_text_to_use")`. Leave as `f` for no transformation. | diff --git a/workflows/raster/copy.yaml b/workflows/raster/copy.yaml index 75e87a7f..5161d566 100644 --- a/workflows/raster/copy.yaml +++ b/workflows/raster/copy.yaml @@ -68,6 +68,12 @@ spec: - '--no-clobber' - '--force' - '--force-no-clobber' + - name: flatten + description: Flatten the files in the target location + default: 'false' + enum: + - 'true' + - 'false' - name: group value: '1000' - name: group_size @@ -92,6 +98,7 @@ spec: - name: target - name: include - name: exclude + - name: flatten - name: group - name: group_size dag: @@ -110,6 +117,8 @@ spec: value: '{{inputs.parameters.include}}' - name: exclude value: '{{inputs.parameters.exclude}}' + - name: flatten + value: '{{inputs.parameters.flatten}}' - name: group value: '{{inputs.parameters.group}}' - name: group_size