From 4529a1d9a760b6754c93d22fd3ad1cc34263d7d8 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Mon, 18 Sep 2023 14:08:56 +1200 Subject: [PATCH 1/2] fix(cogify): lower concurrency and set resource limits to memory = 2x cpu --- workflows/basemaps/imagery-import-cogify.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/workflows/basemaps/imagery-import-cogify.yml b/workflows/basemaps/imagery-import-cogify.yml index 0aaa2b5b..7e3a6c28 100644 --- a/workflows/basemaps/imagery-import-cogify.yml +++ b/workflows/basemaps/imagery-import-cogify.yml @@ -226,11 +226,9 @@ spec: container: resources: requests: - memory: 15.6Gi - cpu: 15000m - ephemeral-storage: 100Gi - limits: - memory: 15.6Gi + memory: 30Gi + cpu: 16000m + ephemeral-storage: 98Gi # 2 pods per 200GB of storage image: ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }} command: [node, /app/node_modules/.bin/cogify] env: @@ -239,6 +237,7 @@ spec: args: - "create" - "--from-file={{= inputs.artifacts.covering_grouped.path }}{{inputs.parameters.covering_grouped_id}}.json" + - "--concurrency=2" # Create a basemaps configuration file to view the imagery - name: create-config From e53808d95d3fb79d678dacbd5949c5669003d5b0 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Mon, 18 Sep 2023 14:09:57 +1200 Subject: [PATCH 2/2] docs: explain why memory was chosen --- workflows/basemaps/imagery-import-cogify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/basemaps/imagery-import-cogify.yml b/workflows/basemaps/imagery-import-cogify.yml index 7e3a6c28..d065c1d7 100644 --- a/workflows/basemaps/imagery-import-cogify.yml +++ b/workflows/basemaps/imagery-import-cogify.yml @@ -227,7 +227,7 @@ spec: resources: requests: memory: 30Gi - cpu: 16000m + cpu: 15000m # AWS gives 2x cpu cores = memory for most instances ephemeral-storage: 98Gi # 2 pods per 200GB of storage image: ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }} command: [node, /app/node_modules/.bin/cogify]