feat(launch): support kaniko builds with user provided pvc and docker…

…config (#84)
wandb · Mar 6, 2024 · 5962b88 · 5962b88
1 parent e72f504
commit 5962b88
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 4 deletions.
diff --git a/charts/launch-agent/Chart.yaml b/charts/launch-agent/Chart.yaml
@@ -3,7 +3,7 @@ name: launch-agent
 icon: https://em-content.zobj.net/thumbs/240/apple/354/rocket_1f680.png
 description: A Helm chart for running the W&B Launch Agent in Kubernetes
 type: application
-version: 0.12.3
+version: 0.13.0
 maintainers:
   - name: wandb
     email: support@wandb.com

diff --git a/charts/launch-agent/README.md b/charts/launch-agent/README.md
@@ -61,3 +61,5 @@ The table below describes all the available variables in the chart:
 | `additionalEnvVars`            | map(string)     | No                   | {}                              | Map with environment variables to be set in the Launch Agent pod.                                                                                |
 | `additionalSecretEnvVars`      | map(string)     | No                   | {}                              | Map with environment variables to be stored in the `launch-agent-secret-env-vars` secret and set in the Launch Agent Pod                         |
 | `customCABundle`               | object          | No                   | {}                              | ConfigMap name and key with the CA Bundle content                                                                                                |
+| `kanikoPvcName`                | string          | No                   | ""                              | Name of a PVC to pass build contexts from the agent to kaniko build containers.                                                                  |
+| `kanikoDockerConfigSecret`     | string          | No                   | ""                              | Name of a kubernetes.io/dockerconfigjson secret that will be mounted in kaniko containers to grant access to private registries.                 |
diff --git a/charts/launch-agent/templates/deployment.yaml b/charts/launch-agent/templates/deployment.yaml
@@ -23,6 +23,19 @@ spec:
 {{- end }}
     spec:
       serviceAccountName: wandb-launch-serviceaccount-{{ .Release.Name }}
+      {{- if .Values.kanikoPvcName }}
+      initContainers:
+      - name: kaniko-volume-chown
+        image: {{ .Values.agent.image }}
+        command: ["sh", "-c"]
+        args:
+        - chown -R 1000:1000 /home/launch_agent/kaniko
+        securityContext:
+          runAsUser: 0
+        volumeMounts:
+          - name: kaniko-pvc
+            mountPath: /home/launch_agent/kaniko
+      {{- end }}
       containers:
         - name: launch-agent
           image: {{ .Values.agent.image }}
@@ -77,6 +90,16 @@ spec:
             - name: REQUESTS_CA_BUNDLE
               value: /usr/local/share/ca-certificates/custom-ca.crt
             {{- end }}
+            {{- if .Values.kanikoPvcName }}
+            - name: WANDB_LAUNCH_KANIKO_PVC_NAME
+              value: {{ .Values.kanikoPvcName }}
+            - name: WANDB_LAUNCH_KANIKO_PVC_MOUNT_PATH
+              value: /home/launch_agent/kaniko
+            {{- end }}
+            {{- if .Values.kanikoDockerConfigSecret }}
+            - name: WANDB_LAUNCH_KANIKO_AUTH_SECRET
+              value: {{ .Values.kanikoDockerConfigSecret }}
+            {{- end }}
           volumeMounts:
             - name: wandb-launch-config
               mountPath: /home/launch_agent/.config/wandb
@@ -92,6 +115,10 @@ spec:
               subPath: custom-ca.crt
               readOnly: true
             {{ end }}
+            {{- if .Values.kanikoPvcName }}
+            - name: kaniko-pvc
+              mountPath: /home/launch_agent/kaniko
+            {{ end }}
       volumes:
         - name: wandb-launch-config
           configMap:
@@ -112,9 +139,15 @@ spec:
               - key: {{ .Values.customCABundle.configMap.key }}
                 path: custom-ca.crt
         {{- end}}
+        {{ if .Values.kanikoPvcName }}
+        - name: kaniko-pvc
+          persistentVolumeClaim:
+            claimName: {{ .Values.kanikoPvcName }}
+        {{- end}}
       nodeSelector:
         {{- toYaml .Values.agent.nodeSelector | nindent 8 }}
 ---
+{{- if .Capabilities.APIVersions.Has "policy/v1" }}
 apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
@@ -124,4 +157,5 @@ spec:
   selector:
     matchLabels:
       app: launch-agent-{{ .Release.Name }}
----
+---
+{{- end }}
diff --git a/charts/launch-agent/values.yaml b/charts/launch-agent/values.yaml
@@ -41,12 +41,14 @@ launchConfig: |
     type: noop
 
 # # Additional environment variables to set on the agent container.
-additionalEnvVars: {}
+additionalEnvVars:
+  {}
   # - name: MY_ENV_VAR
   #   value: "my-value"
 
 # # Additional secret environment variables to set on the agent container.
-additionalSecretEnvVars: {}
+additionalSecretEnvVars:
+  {}
   # - name: MY_SECRET_ENV_VAR
   #   value: "my-secret-value"
 
@@ -71,3 +73,12 @@ customCABundle:
   configMap:
     name:
     key:
+
+# Configure the agent to pass build contexts to kaniko through a persistent volume claim.
+# The pvc will be mounted at /kaniko in the agent container and at /context the kaniko containers
+# that it starts.
+kanikoPvcName:
+
+# Name of a secret containing a docker config.json file to use with kaniko.
+# This secret will be mounted at /kaniko/.docker in the agent container.
+kanikoDockerConfigSecret: