feat: add prometheus operator to fullstack-cluster-setup (#460)

Signed-off-by: Jeromy Cannon <jeromy@swirldslabs.com>

.github/workflows/zxc-compile-code.yaml

@@ -118,12 +118,6 @@ jobs:
         with:
           version: "v3.12.3" # helm version
 
-      - name: Setup Kubernetes Operators
-        working-directory: dev
-        if: ${{ (inputs.enable-unit-tests || inputs.enable-nodejs-tests) && !cancelled() && !failure() }}
-        run: |
-          make deploy-prometheus-operator
-
       # Technically, this step is not required for the unit tests to run, but it is useful for debugging setup issues.
       - name: Kubernetes Cluster Info
         if: ${{ (inputs.enable-unit-tests || inputs.enable-nodejs-tests) && !cancelled() && !failure() }}

charts/fullstack-cluster-setup/Chart.lock

@@ -2,5 +2,11 @@ dependencies:
 - name: operator
   repository: https://operator.min.io/
   version: 5.0.7
-digest: sha256:2f7b3843ce9b2417b2db1a7af249a6c4d097cb13e65d557c0c80a55688e1a172
-generated: "2023-10-25T15:14:25.580221+01:00"
+- name: kube-prometheus-stack
+  repository: https://prometheus-community.github.io/helm-charts
+  version: 52.0.1
+- name: grafana-agent
+  repository: https://grafana.github.io/helm-charts
+  version: 0.27.1
+digest: sha256:1d98a6cef658a43c687cf9cfa773f8cb7f5b2468b7e38e9cbb4f99cbcfec6651
+generated: "2023-10-27T14:52:43.529303+01:00"

charts/fullstack-cluster-setup/Chart.yaml

@@ -29,3 +29,14 @@ dependencies:
     version: 5.0.7
     repository: https://operator.min.io/
     condition: cloud.minio.enable
+
+  - name: kube-prometheus-stack
+    alias: prometheus-stack
+    version: 52.0.1
+    repository: https://prometheus-community.github.io/helm-charts
+    condition: cloud.prometheusStack.enable
+
+  - name: grafana-agent
+    version: 0.27.1
+    repository: https://grafana.github.io/helm-charts
+    condition: cloud.grafanaAgent.enable

charts/fullstack-cluster-setup/values.yaml

@@ -11,3 +11,7 @@ global:
 cloud:
   minio:
     enable: false
+  prometheusStack:
+    enable: false
+  grafanaAgent:
+    enable: false

dev/Makefile

@@ -63,16 +63,15 @@ update-helm-dependencies:
 
 .PHONY: init-shared
 init-shared:
-	echo "# values override file for fullstack-cluster-setup" > ${CLUSTER_SETUP_VALUES_FILE}
+	echo "cloud:" > ${CLUSTER_SETUP_VALUES_FILE}
 
 .PHONY: deploy-shared
-deploy-shared: init-shared update-helm-dependencies deploy-gateway-api deploy-prometheus-operator enable-minio-operator-if-required
+deploy-shared: init-shared update-helm-dependencies deploy-gateway-api enable-prometheus-stack-if-required enable-minio-operator-if-required
 	source "${SCRIPTS_DIR}/main.sh" && deploy_shared # run only after gateway-api CRDs are available
 
 .PHONY: destroy-shared
 destroy-shared:
 	source "${SCRIPTS_DIR}/main.sh" && destroy_shared
-	-$(MAKE) destroy-prometheus-operator
 
 .PHONY: deploy-chart
 deploy-chart:
@@ -165,13 +164,21 @@ destroy-gateway-api:
 	source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && destroy_envoy_gateway_api
 
 ######################################### Telemetry #################################
-.PHONY: deploy-prometheus-operator
-deploy-prometheus-operator:
-	source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && deploy-prometheus-operator
+.PHONY: enable-prometheus-stack-if-required
+enable-prometheus-stack-if-required:
+	@$(MAKE) is-prometheus-operator-installed || $(MAKE) enable-prometheus-stack
+
+.PHONY: enable-prometheus-stack
+enable-prometheus-stack:
+	@echo ">> Enabling prometheus stack..."; \
+	echo "  prometheusStack:" >> ${CLUSTER_SETUP_VALUES_FILE}; \
+	echo "    enable: true" >> ${CLUSTER_SETUP_VALUES_FILE};
 
-.PHONY: destroy-prometheus-operator
-destroy-prometheus-operator:
-	source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && destroy-prometheus-operator
+.PHONY: is-prometheus-operator-installed
+is-prometheus-operator-installed:
+	@echo ">> Checking for prometheus operator..."; \
+	kubectl get pods --all-namespaces | grep "prometheus-operator\|pr-operator" &> /dev/null && { echo "Found prometheus operator."; exit 0; }; \
+	echo "Prometheus operator not found."; exit 1
 
 .PHONY: deploy-grafana-tempo
 deploy-grafana-tempo:
@@ -182,12 +189,19 @@ destroy-grafana-tempo:
 	source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && unexpose_grafana && destroy_grafana_tempo
 
 .PHONY: deploy-prometheus
-deploy-prometheus: deploy-prometheus-operator
+deploy-prometheus: init-shared deploy-gateway-api enable-prometheus-stack
+	source "${SCRIPTS_DIR}/main.sh" && deploy_shared # run only after gateway-api CRDs are available
+	-$(MAKE) wait-for-prometheus-operator
 	source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && deploy-prometheus && expose_prometheus
 
+.PHONY: wait-for-prometheus-operator
+wait-for-prometheus-operator:
+	kubectl wait --for=condition=Ready pods -l app.kubernetes.io/name=prometheus-operator --timeout 300s --all-namespaces
+
 .PHONY: destroy-prometheus
 destroy-prometheus:
 	source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && unexpose_prometheus && destroy-prometheus
+	-$(MAKE) destroy-shared
 
 .PHONY: deploy-telemetry-stack
 deploy-telemetry-stack: deploy-prometheus deploy-grafana-tempo
@@ -202,7 +216,6 @@ destroy-telemetry-stack:
 .PHONY: enable-minio-operator
 enable-minio-operator:
 	@echo ">> Enabling minio operator..."; \
-	echo "cloud:" >> ${CLUSTER_SETUP_VALUES_FILE}; \
 	echo "  minio:" >> ${CLUSTER_SETUP_VALUES_FILE}; \
 	echo "    enable: true" >> ${CLUSTER_SETUP_VALUES_FILE};