feat: add labels at the podMetadata level TDE-1249 (#724)

#### Motivation Managing pod labels helps to do some kind of filtering/analytics on the pod runs and improve logging information. #### Modification - populates the pod labels using `podMetadata` at the workflow spec level. Workflow `templates` that are called, get their pods to get these labels too. #### Checklist - [ ] Tests updated - no tests - but ran manual ones - [ ] Docs updated - no docs - [X] Issue linked in Title
linz · Aug 27, 2024 · f0bef77 · f0bef77
1 parent 253853a
commit f0bef77
Show file tree

Hide file tree

Showing 21 changed files with 84 additions and 1 deletion.
diff --git a/workflows/basemaps/create-config.yaml b/workflows/basemaps/create-config.yaml
@@ -8,6 +8,9 @@ metadata:
     linz.govt.nz/category: basemaps
 spec:
   entrypoint: main
+  podMetadata:
+    labels:
+      linz.govt.nz/category: basemaps
   arguments:
     parameters:
       - name: version_basemaps_cli

diff --git a/workflows/basemaps/create-overview-all.yaml b/workflows/basemaps/create-overview-all.yaml
@@ -13,6 +13,10 @@ spec:
   nodeSelector:
     karpenter.sh/capacity-type: 'spot'
   entrypoint: main
+  podMetadata:
+    labels:
+      linz.govt.nz/category: basemaps
+      linz.govt.nz/data-type: raster
   arguments:
     parameters:
       - name: version_basemaps_cli

diff --git a/workflows/basemaps/create-overview.yaml b/workflows/basemaps/create-overview.yaml
@@ -10,6 +10,10 @@ metadata:
     linz.govt.nz/data-type: raster
 spec:
   entrypoint: main
+  podMetadata:
+    labels:
+      linz.govt.nz/category: basemaps
+      linz.govt.nz/data-type: raster
   arguments:
     parameters:
       - name: version_basemaps_cli

diff --git a/workflows/basemaps/imagery-import-cogify.yml b/workflows/basemaps/imagery-import-cogify.yml
@@ -25,6 +25,12 @@ spec:
         expression: workflow.parameters.ticket
       linz.govt.nz/region:
         expression: workflow.parameters.region
+  podMetadata:
+    labels:
+      linz.govt.nz/category: basemaps
+      linz.govt.nz/data-type: raster
+      linz.govt.nz/ticket: '{{workflow.parameters.ticket}}'
+      linz.govt.nz/region: '{{workflow.parameters.region}}'
   arguments:
     parameters:
       - name: version_basemaps_cli
@@ -75,7 +81,7 @@ spec:
       - name: source
         description: Source imagery location "s3://linz-imagery"
         value: 's3://linz-imagery-staging/test/sample/'
-      
+
       - name: require_stac_collection
         description: Validate that a STAC collection.json exists with the source
         value: 'true'

diff --git a/workflows/basemaps/mapsheet-json.yaml b/workflows/basemaps/mapsheet-json.yaml
@@ -10,6 +10,11 @@ metadata:
     linz.govt.nz/data-sub-type: maps
 spec:
   entrypoint: main
+  podMetadata:
+    labels:
+      linz.govt.nz/category: basemaps
+      linz.govt.nz/data-type: raster
+      linz.govt.nz/data-sub-type: maps
   arguments:
     parameters:
       - name: version_argo_tasks

diff --git a/workflows/basemaps/vector-etl.yaml b/workflows/basemaps/vector-etl.yaml
@@ -9,6 +9,10 @@ metadata:
     linz.govt.nz/data-type: vector
 spec:
   entrypoint: main
+  podMetadata:
+    labels:
+      linz.govt.nz/category: basemaps
+      linz.govt.nz/data-type: vector
   arguments:
     parameters:
       - name: version_argo_tasks

diff --git a/workflows/cron/cron-stac-validate-fast.yaml b/workflows/cron/cron-stac-validate-fast.yaml
@@ -15,6 +15,9 @@ spec:
   suspend: false
   workflowSpec:
     entrypoint: main
+    podMetadata:
+      labels:
+        linz.govt.nz/category: stac
     arguments:
       parameters:
         - name: checksum_assets

diff --git a/workflows/cron/cron-stac-validate-full.yaml b/workflows/cron/cron-stac-validate-full.yaml
@@ -15,6 +15,9 @@ spec:
   suspend: false
   workflowSpec:
     entrypoint: main
+    podMetadata:
+      labels:
+        linz.govt.nz/category: stac
     arguments:
       parameters:
         - name: version_argo_tasks

diff --git a/workflows/cron/cron-vector-etl-roads-addressing.yaml b/workflows/cron/cron-vector-etl-roads-addressing.yaml
@@ -15,6 +15,10 @@ spec:
   failedJobsHistoryLimit: 3
   suspend: false
   workflowSpec:
+    podMetadata:
+      labels:
+        linz.govt.nz/category: basemaps
+        linz.govt.nz/data-type: vector
     workflowTemplateRef:
       name: basemaps-vector-etl
     arguments:

diff --git a/workflows/cron/cron-vector-etl.yaml b/workflows/cron/cron-vector-etl.yaml
@@ -15,6 +15,10 @@ spec:
   failedJobsHistoryLimit: 3
   suspend: false
   workflowSpec:
+    podMetadata:
+      labels:
+        linz.govt.nz/category: basemaps
+        linz.govt.nz/data-type: vector
     workflowTemplateRef:
       name: basemaps-vector-etl
     arguments:

diff --git a/workflows/raster/copy.yaml b/workflows/raster/copy.yaml
@@ -23,6 +23,12 @@ spec:
         expression: workflow.parameters.ticket
       linz.govt.nz/region:
         expression: workflow.parameters.region
+  podMetadata:
+    labels:
+      linz.govt.nz/category: raster
+      linz.govt.nz/data-type: raster
+      linz.govt.nz/ticket: '{{workflow.parameters.ticket}}'
+      linz.govt.nz/region: '{{workflow.parameters.region}}'
   arguments:
     parameters:
       - name: version_argo_tasks

diff --git a/workflows/raster/publish-odr.yaml b/workflows/raster/publish-odr.yaml
@@ -23,6 +23,12 @@ spec:
         expression: workflow.parameters.ticket
       linz.govt.nz/region:
         expression: workflow.parameters.region
+  podMetadata:
+    labels:
+      linz.govt.nz/category: raster
+      linz.govt.nz/data-type: raster
+      linz.govt.nz/ticket: '{{workflow.parameters.ticket}}'
+      linz.govt.nz/region: '{{workflow.parameters.region}}'
   arguments:
     parameters:
       - name: version_argo_tasks

diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml
@@ -24,6 +24,12 @@ spec:
         expression: workflow.parameters.ticket
       linz.govt.nz/region:
         expression: workflow.parameters.region
+  podMetadata:
+    labels:
+      linz.govt.nz/category: raster
+      linz.govt.nz/data-type: raster
+      linz.govt.nz/ticket: '{{workflow.parameters.ticket}}'
+      linz.govt.nz/region: '{{workflow.parameters.region}}'
   arguments:
     parameters:
       - name: version_argo_tasks

diff --git a/workflows/raster/tests.yaml b/workflows/raster/tests.yaml
@@ -12,6 +12,10 @@ spec:
   podGC:
     strategy: OnPodCompletion # Delete pod once its finished
   entrypoint: test-script
+  podMetadata:
+    labels:
+      linz.govt.nz/category: test
+      linz.govt.nz/data-type: raster
   arguments:
     parameters:
       - name: version_topo_imagery

diff --git a/workflows/stac/stac-validate-parallel.yaml b/workflows/stac/stac-validate-parallel.yaml
@@ -11,6 +11,9 @@ spec:
   nodeSelector:
     karpenter.sh/capacity-type: 'spot'
   entrypoint: main
+  podMetadata:
+    labels:
+      linz.govt.nz/category: stac
   arguments:
     parameters:
       - name: version_argo_tasks

diff --git a/workflows/test/env.yaml b/workflows/test/env.yaml
@@ -8,6 +8,9 @@ metadata:
     linz.govt.nz/category: test
 spec:
   entrypoint: env
+  podMetadata:
+    labels:
+      linz.govt.nz/category: test
   templates:
     - name: env
       container:

diff --git a/workflows/test/flatten.yaml b/workflows/test/flatten.yaml
@@ -10,6 +10,9 @@ spec:
   nodeSelector:
     karpenter.sh/capacity-type: 'spot'
   entrypoint: main
+  podMetadata:
+    labels:
+      linz.govt.nz/category: test
   arguments:
     parameters:
       - name: version_argo_tasks

diff --git a/workflows/test/generate-path.yaml b/workflows/test/generate-path.yaml
@@ -10,6 +10,9 @@ spec:
   nodeSelector:
     karpenter.sh/capacity-type: 'spot'
   entrypoint: main
+  podMetadata:
+    labels:
+      linz.govt.nz/category: test
   arguments:
     parameters:
       - name: version_argo_tasks

diff --git a/workflows/test/list.arm.yaml b/workflows/test/list.arm.yaml
@@ -9,6 +9,9 @@ metadata:
     linz.govt.nz/category: test
 spec:
   entrypoint: main
+  podMetadata:
+    labels:
+      linz.govt.nz/category: test
   arguments:
     parameters:
       - name: version_argo_tasks

diff --git a/workflows/test/list.yaml b/workflows/test/list.yaml
@@ -9,6 +9,9 @@ metadata:
 spec:
   serviceAccountName: workflow-runner-sa
   entrypoint: main
+  podMetadata:
+    labels:
+      linz.govt.nz/category: test
   arguments:
     parameters:
       - name: version_argo_tasks

diff --git a/workflows/test/sleep.yml b/workflows/test/sleep.yml
@@ -8,6 +8,9 @@ metadata:
     linz.govt.nz/category: test
 spec:
   entrypoint: sleep
+  podMetadata:
+    labels:
+      linz.govt.nz/category: test
   templates:
     - name: sleep
       nodeSelector: