chore: add redis and parquet

.gitignore

@@ -1,3 +1,4 @@
 *.tgz
 .vscode/
 dryrun.yaml
+test-values.yaml

charts/operator-wandb/Chart.lock

@@ -8,8 +8,14 @@ dependencies:
 - name: weave
   repository: file://charts/weave
   version: 0.1.0
+- name: parquet
+  repository: file://charts/parquet
+  version: 0.1.0
 - name: mysql
   repository: file://charts/mysql
   version: 0.1.0
-digest: sha256:97fa6018a524348b546cecd7a7c49f872eac0040698e833f4b43ed583ba069fa
-generated: "2023-08-15T22:43:20.053539432-04:00"
+- name: redis
+  repository: file://charts/redis
+  version: 0.1.0
+digest: sha256:1b7b3d43935aec297fae6fc09416cd5df785852f3d4d6ddd22617efe0fc20ca7
+generated: "2023-08-21T13:31:32.01111468-04:00"

charts/operator-wandb/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: operator-wandb
 description: A Helm chart for deploying W&B to Kubernetes
 type: application
-version: 0.1.1
+version: 0.1.2
 appVersion: 1.0.0
 icon: https://wandb.ai/logo.svg
 maintainers:
@@ -23,7 +23,15 @@ dependencies:
     version: "*.*.*"
     repository: file://charts/weave
     condition: weave.install
+  - name: parquet
+    version: "*.*.*"
+    repository: file://charts/parquet
+    condition: parquet.install
   - name: mysql
     version: "*.*.*"
     repository: file://charts/mysql
     condition: mysql.install
+  - name: redis
+    version: "*.*.*"
+    repository: file://charts/redis
+    condition: redis.install

charts/operator-wandb/charts/app/templates/deployment.yaml

@@ -74,6 +74,7 @@ spec:
             - name: GORILLA_CORS_ORIGINS
               value: "{{ join "," .Values.extraCors }}"
             {{- end }}
+
             - name: MYSQL_PORT
               value: "{{ include "wandb.mysql.port" . }}"
             - name: MYSQL_HOST
@@ -87,12 +88,38 @@ spec:
                 secretKeyRef:
                   name: {{ include "wandb.mysql.passwordSecret" . }}
                   key:  MYSQL_PASSWORD
-            # - name: MYSQL
-            #   value: "mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@$(MYSQL_HOST):$(MYSQL_PORT)/$(MYSQL_DATABASE)"
-            - name: HOST
-              value: {{ .Values.host }}
+            - name: MYSQL
+              value: "mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@$(MYSQL_HOST):$(MYSQL_PORT)/$(MYSQL_DATABASE)"
+
+            - name: REDIS_PORT
+              value: "{{ include "wandb.redis.port" . }}"
+            {{- if ne (include "wandb.redis.password" .) "" }}
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "wandb.redis.passwordSecret" . }}
+                  key: REDIS_PASSWORD
+            {{- end }}
+            - name: REDIS_HOST
+              value: "{{ include "wandb.redis.host" . }}"
+
             - name: WEAVE_SERVICE
               value: {{ .Release.Name }}-weave:9994
+
+            - name: PARQUET_HOST
+              value: "http://{{ .Release.Name }}-parquet:8087"
+            - name: PARQUET_ENABLED
+              value: "true"
+
+
+            - name: HOST
+              value: {{ .Values.host }}
+            - name: BUCKET
+              value: {{ .Values.global.storage.connectionString }}
+            - name: AWS_REGION
+              value: {{ .Values.global.storage.region }}
+            - name: LOGGING_ENABLED
+              value: 'true'
             {{- include "app.extraEnv" (dict "global" .Values.global "local" .) | nindent 12 }}
             {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}
           livenessProbe:

charts/operator-wandb/charts/mysql/templates/pvc.yaml

@@ -10,7 +10,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: {{ include "wandb.storageClass" . }}
+  {{ include "wandb.storageClass" . | nindent 4 }}
   resources:
     requests:
       storage: {{ .Values.persistence.size | quote }}

charts/operator-wandb/charts/mysql/values.yaml

@@ -17,7 +17,6 @@ tolerations: []
 extraEnv: {}
 extraEnvFrom: {}
 
-extraCors: []
 common: {}
 
 service:

charts/operator-wandb/charts/parquet/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/operator-wandb/charts/parquet/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: parquet
+description: A Helm chart for deploying parquet to kubernetes
+type: application
+version: 0.1.0
+appVersion: 1.0.0

charts/operator-wandb/charts/parquet/templates/_helpers.tpl

@@ -0,0 +1,101 @@
+{{/* vim: set filetype=mustache: */}}
+
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "parquet.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "parquet.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "parquet.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "parquet.labels" -}}
+helm.sh/chart: {{ include "parquet.chart" . }}
+{{ include "parquet.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+wandb.com/app-name: {{ include "parquet.chart" . }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "parquet.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "parquet.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "parquet.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "parquet.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Returns the extraEnv keys and values to inject into containers.
+
+Global values will override any chart-specific values.
+*/}}
+{{- define "parquet.extraEnv" -}}
+{{- $allExtraEnv := merge (default (dict) .local.extraEnv) .global.extraEnv -}}
+{{- range $key, $value := $allExtraEnv }}
+- name: {{ $key }}
+  value: {{ $value | quote }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Returns a list of _common_ labels to be shared across all
+app deployments and other shared objects.
+*/}}
+{{- define "parquet.commonLabels" -}}
+{{- $commonLabels := default (dict) .Values.common.labels -}}
+{{- if $commonLabels }}
+{{-   range $key, $value := $commonLabels }}
+{{ $key }}: {{ $value | quote }}
+{{-   end }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Returns a list of _pod_ labels to be shared across all
+app deployments.
+*/}}
+{{- define "parquet.podLabels" -}}
+{{- range $key, $value := .Values.pod.labels }}
+{{ $key }}: {{ $value | quote }}
+{{- end }}
+{{- end -}}

charts/operator-wandb/charts/parquet/templates/deployment.yaml

@@ -0,0 +1,68 @@
+{{- if .Values.enabled }}
+{{- $imageCfg := dict "global" $.Values.global.image "local" $.Values.image -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "parquet.fullname" . }}
+  labels:
+    {{- include "wandb.commonLabels" . | nindent 4 }}
+    {{- include "parquet.commonLabels" . | nindent 4 }}
+    {{- include "parquet.labels" . | nindent 4 }}
+    {{- if .Values.deployment.labels -}}
+    {{-   toYaml .Values.deployment.labels | nindent 4 }}
+    {{- end }}
+  annotations:
+    {{- include "wandb.deploymentAnnotations" $ | nindent 4 }}
+    {{- if .Values.deployment.annotations -}}
+    {{-   toYaml .Values.deployment.annotations | nindent 4 }}
+    {{- end }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "wandb.selectorLabels" $ | nindent 6 }}
+      {{- include "parquet.labels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "wandb.podLabels" . | nindent 8 }}
+        {{- include "parquet.commonLabels" . | nindent 8 }}
+        {{- include "parquet.podLabels" . | nindent 8 }}
+        {{- include "parquet.labels" . | nindent 8 }}
+      annotations:
+        {{- if .Values.pod.annotations -}}
+        {{-   toYaml .Values.pod.annotations | nindent 4 }}
+        {{- end }}
+    spec:
+      {{- if .tolerations }}
+      tolerations:
+        {{- toYaml .tolerations | nindent 8 }}
+      {{- end }}
+      {{- include "wandb.nodeSelector" . | nindent 6 }}
+      {{- include "wandb.priorityClassName" . | nindent 6 }}
+      {{- include "wandb.podSecurityContext" .Values.pod.securityContext | nindent 6 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          ports:
+            - name: parquet
+              containerPort: 8087
+              protocol: TCP
+          env:
+            - name: ONLY_SERVICE
+              value: gorilla-parquet
+
+            - name: BUCKET
+              value: {{ .Values.global.storage.connectionString }}
+            - name: AWS_REGION
+              value: {{ .Values.global.storage.region }}
+
+            - name: LOGGING_ENABLED
+              value: 'true'
+
+            {{- include "parquet.extraEnv" (dict "global" .Values.global "local" .) | nindent 12 }}
+            {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}
+
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+{{- end }}

charts/operator-wandb/charts/parquet/templates/service.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "weave.fullname" . }}
+  labels:
+    {{- include "wandb.commonLabels" . | nindent 4 }}
+    {{- include "weave.labels" . | nindent 4 }}
+    {{- include "weave.commonLabels" . | nindent 4 }}
+    {{- if .Values.service.labels -}}
+    {{-   toYaml .Values.service.labels | nindent 4 }}
+    {{- end }}
+  annotations:
+    {{- if .Values.service.annotations -}}
+    {{-   toYaml .Values.service.annotations | nindent 4 }}
+    {{- end }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: 8087
+      protocol: TCP
+      name: parquet
+  selector:
+    {{- include "weave.labels" . | nindent 4 }}
+{{- end }}

charts/operator-wandb/charts/parquet/values.yaml

@@ -0,0 +1,42 @@
+enabled: true
+
+nameOverride: ""
+fullnameOverride: ""
+
+image:
+  repository: wandb/local
+  tag: latest
+  pullPolicy: Always
+
+tolerations: []
+
+extraEnv: {}
+extraEnvFrom: {}
+
+extraCors: []
+
+common: {}
+deployment: {}
+serviceAccount: {}
+clusterRole: {}
+
+service:
+  type: ClusterIP
+
+pod:
+  securityContext:
+    runAsUser: 1000
+    fsGroup: 1000
+
+resources:
+  # We usually recommend not to specify default resources and to leave this as a
+  # conscious choice for the user. This also increases chances charts run on
+  # environments with little resources, such as Minikube. If you do want to
+  # specify resources, uncomment the following lines, adjust them as necessary,
+  # and remove the curly braces after 'resources:'.
+  requests:
+    cpu: 1
+    memory: 2Gi
+  limits:
+    cpu: 8
+    memory: 16Gi