diff --git a/deployment/k8/migration-assistant/Chart.lock b/deployment/k8/migration-assistant/Chart.lock index cdb8f2444..4f6677b16 100644 --- a/deployment/k8/migration-assistant/Chart.lock +++ b/deployment/k8/migration-assistant/Chart.lock @@ -1,9 +1,15 @@ dependencies: +- name: opensearch + repository: https://opensearch-project.github.io/helm-charts/ + version: 2.23.1 +- name: kafka-op + repository: file://../services/kafka-op + version: 0.1.0 - name: migration-console repository: file://../services/migration-console version: 0.1.0 - name: capture-proxy repository: file://../services/capture-proxy version: 0.1.0 -digest: sha256:7897ada3193d874398b7fdc67a8f45eeb650302c190fe07ef9b29494d678e2f2 -generated: "2024-10-23T20:35:14.098691-05:00" +digest: sha256:3dcd125d6e968ddfc212318128d5f4222822bf33d0593f061f3c2eed40cea144 +generated: "2024-10-24T15:36:04.42393-05:00" diff --git a/deployment/k8/migration-assistant/Chart.yaml b/deployment/k8/migration-assistant/Chart.yaml index eaa4f99a8..b316df4c7 100644 --- a/deployment/k8/migration-assistant/Chart.yaml +++ b/deployment/k8/migration-assistant/Chart.yaml @@ -5,21 +5,26 @@ dependencies: # - name: elasticsearch # version: "0.1.0" # repository: "file://../services/elasticsearch" -# - name: opensearch-helm -# version: "0.1.0" -# repository: "file://../services/opensearch-helm" -# - name: kafka-op -# version: "0.1.0" -# repository: "file://../services/kafka-op" + - name: opensearch + version: "2.23.1" + repository: "https://opensearch-project.github.io/helm-charts/" + alias: opensearch-helm + condition: opensearch-helm-enabled + - name: kafka-op + version: "0.1.0" + repository: "file://../services/kafka-op" + condition: kafka-op-enabled - name: migration-console version: "0.1.0" repository: "file://../services/migration-console" + condition: migration-console-enabled # - name: replayer # version: "0.1.0" # repository: "file://../services/replayer" - name: capture-proxy version: "0.1.0" repository: "file://../services/capture-proxy" + condition: capture-proxy-enabled # - name: reindex-from-snapshot # version: "0.1.0" # repository: "file://../services/reindex-from-snapshot" @@ -29,12 +34,3 @@ dependencies: # - name: snapshot-vol # version: "0.1.0" # repository: "file://../services/shared/snapshot-vol" -# - name: jaeger -# version: "0.1.0" -# repository: "file://../services/jaeger" -# - name: prometheus -# version: "0.1.0" -# repository: "file://../services/prometheus" -# - name: grafana -# version: "0.1.0" -# repository: "file://../services/grafana" diff --git a/deployment/k8/migration-assistant/values.yaml b/deployment/k8/migration-assistant/values.yaml index 9c7e9f34d..028ccf449 100644 --- a/deployment/k8/migration-assistant/values.yaml +++ b/deployment/k8/migration-assistant/values.yaml @@ -1,6 +1,11 @@ +# Define all enabled/disabled migration services +migration-console-enabled: false +capture-proxy-enabled: false +kafka-op-enabled: true +opensearch-helm-enabled: false +# Define migration service configuration migration-console: - enabled: true replicaCount: 1 image: repository: migrations/migration_console @@ -36,7 +41,6 @@ migration-console: mountPath: /storage capture-proxy: - enabled: true replicaCount: 1 image: repository: migrations/capture_proxy @@ -63,3 +67,26 @@ capture-proxy: cpu: 100m memory: 128Mi +kafka-op: + clusterName: kafka-cluster + kafkaVersion: 3.8.0 + kafkaMetadataVersion: 3.8-IV0 + nodePool: + replicas: 1 + volume: + size: 10Gi + deleteClaim: true + +opensearch-helm: + fullnameOverride: "opensearch" + image: + tag: "2.17.0" + replicas: 1 + singleNode: true + config: + discovery.type: single-node + extraEnvs: + - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD + value: myStrongPassword123! + service: + type: LoadBalancer diff --git a/deployment/k8/services/capture-proxy/values.yaml b/deployment/k8/services/capture-proxy/values.yaml index 90beddc2e..1af029ff9 100644 --- a/deployment/k8/services/capture-proxy/values.yaml +++ b/deployment/k8/services/capture-proxy/values.yaml @@ -1,3 +1 @@ # Default values for capture proxy. - -enabled: false diff --git a/deployment/k8/services/common-lib/templates/_deployment.yaml b/deployment/k8/services/common-lib/templates/_deployment.yaml index 49b8eb2a9..647ae4d5a 100644 --- a/deployment/k8/services/common-lib/templates/_deployment.yaml +++ b/deployment/k8/services/common-lib/templates/_deployment.yaml @@ -1,5 +1,4 @@ {{- define "common-lib.deployment.yml" -}} -{{- if .Values.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -61,4 +60,3 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end -}} diff --git a/deployment/k8/services/common-lib/templates/_service.yaml b/deployment/k8/services/common-lib/templates/_service.yaml index 2b148ab44..24d7bcea8 100644 --- a/deployment/k8/services/common-lib/templates/_service.yaml +++ b/deployment/k8/services/common-lib/templates/_service.yaml @@ -1,5 +1,4 @@ {{- define "common-lib.service.yml" -}} -{{- if .Values.enabled }} apiVersion: v1 kind: Service metadata: @@ -15,4 +14,3 @@ spec: selector: app: {{ .Chart.Name }} {{- end }} -{{- end -}} diff --git a/deployment/k8/services/grafana/Chart.lock b/deployment/k8/services/grafana/Chart.lock deleted file mode 100644 index 23e9f4fec..000000000 --- a/deployment/k8/services/grafana/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: grafana - repository: https://grafana.github.io/helm-charts - version: 8.5.0 -digest: sha256:9a11b7ae0a252d66e7bb0db397b666f2a205f644aa0d0b2a79db35358e38015c -generated: "2024-10-17T23:02:18.615764-05:00" diff --git a/deployment/k8/services/grafana/Chart.yaml b/deployment/k8/services/grafana/Chart.yaml deleted file mode 100644 index 8ad83f9fe..000000000 --- a/deployment/k8/services/grafana/Chart.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v2 -name: grafana -description: A Helm chart for deploying Grafana -appVersion: "3.5.0" -version: 0.1.0 -dependencies: - - name: grafana - version: "8.5.0" - repository: "https://grafana.github.io/helm-charts" diff --git a/deployment/k8/services/grafana/templates/configmap.yaml b/deployment/k8/services/grafana/templates/configmap.yaml deleted file mode 100644 index 3de2b1bfa..000000000 --- a/deployment/k8/services/grafana/templates/configmap.yaml +++ /dev/null @@ -1,136 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus-metrics-dashboard - labels: - grafana_dashboard: "1" -data: - prometheus-metrics-dashboard.json: | - { - "id": null, - "title": "Prometheus Metrics Dashboard", - "tags": [], - "timezone": "browser", - "schemaVersion": 30, - "version": 1, - "refresh": "5s", - "panels": [ - { - "type": "graph", - "title": "CPU Usage", - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum(rate(node_cpu_seconds_total{mode!=\"idle\"}[5m])) by (instance)", - "legendFormat": "thing", - "refId": "A" - } - ], - "xaxis": { - "mode": "time" - }, - "yaxes": [ - { - "format": "percent", - "label": "CPU Usage" - } - ], - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - } - }, - { - "type": "graph", - "title": "Memory Usage", - "datasource": "Prometheus", - "targets": [ - { - "expr": "node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes", - "legendFormat": "Memory Usage", - "refId": "A" - } - ], - "xaxis": { - "mode": "time" - }, - "yaxes": [ - { - "format": "percent", - "label": "Memory Usage" - } - ], - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - } - }, - { - "type": "graph", - "title": "Disk I/O", - "datasource": "Prometheus", - "targets": [ - { - "expr": "rate(node_disk_io_time_seconds_total[5m])", - "legendFormat": "thing - thing2", - "refId": "A" - } - ], - "xaxis": { - "mode": "time" - }, - "yaxes": [ - { - "format": "s", - "label": "I/O Time" - } - ], - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 8 - } - }, - { - "type": "singlestat", - "title": "Total Requests", - "datasource": "Prometheus", - "targets": [ - { - "expr": "sum(rate(http_requests_total[5m]))", - "refId": "A" - } - ], - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 16 - }, - "format": "none" - }, - { - "type": "singlestat", - "title": "Up Time", - "datasource": "Prometheus", - "targets": [ - { - "expr": "time() - node_boot_time_seconds", - "refId": "A" - } - ], - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 16 - }, - "format": "s" - } - ] - } \ No newline at end of file diff --git a/deployment/k8/services/grafana/values.yaml b/deployment/k8/services/grafana/values.yaml deleted file mode 100644 index 4bc84f74b..000000000 --- a/deployment/k8/services/grafana/values.yaml +++ /dev/null @@ -1,40 +0,0 @@ -grafana: - service: - type: LoadBalancer - port: 3000 - targetPort: 3000 - grafana.ini: - auth: - # Disable login form so anonymous users won't be prompted to log in - disable_login_form: true - auth.anonymous: - # Enable anonymous access - enabled: true - # Set the role for anonymous users (e.g., Viewer, Editor) - org_role: Editor - ## Grafana data sources configuration - datasources: - datasources.yaml: - apiVersion: 1 - datasources: - - name: Prometheus - type: prometheus - access: proxy - url: http://prometheus-server - isDefault: true - editable: true - - name: Jaeger - type: jaeger - access: proxy - url: http://jaeger-query:16686 - isDefault: false - editable: true - - ## Set up the sidecar to import data sources (usually enabled by default) - sidecar: - datasources: - enabled: true - dashboards: - enabled: true - label: grafana_dashboard - diff --git a/deployment/k8/services/jaeger/Chart.lock b/deployment/k8/services/jaeger/Chart.lock deleted file mode 100644 index 1e746c04d..000000000 --- a/deployment/k8/services/jaeger/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: jaeger - repository: https://jaegertracing.github.io/helm-charts - version: 3.2.0 -digest: sha256:41ac5fc4455e3fbdd08464f77e94d308ccab23dfde20a1c8f44b168bcbc38194 -generated: "2024-10-18T11:21:05.929732-05:00" diff --git a/deployment/k8/services/jaeger/Chart.yaml b/deployment/k8/services/jaeger/Chart.yaml deleted file mode 100644 index ebfcc5be9..000000000 --- a/deployment/k8/services/jaeger/Chart.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v2 -name: jaeger -description: A Helm chart for deploying Jaeger -appVersion: "3.5.0" -version: 0.1.0 -dependencies: - - name: jaeger - version: "3.2.0" - repository: "https://jaegertracing.github.io/helm-charts" diff --git a/deployment/k8/services/jaeger/values.yaml b/deployment/k8/services/jaeger/values.yaml deleted file mode 100644 index f1658bb75..000000000 --- a/deployment/k8/services/jaeger/values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -jaeger: - allInOne: - enabled: true - provisionDataStore: - cassandra: false - storage: - type: memory - agent: - enabled: false - collector: - enabled: false - query: - enabled: false diff --git a/deployment/k8/services/kafka-op/templates/kafka-single-node.yaml b/deployment/k8/services/kafka-op/templates/kafka-cluster.yaml similarity index 74% rename from deployment/k8/services/kafka-op/templates/kafka-single-node.yaml rename to deployment/k8/services/kafka-op/templates/kafka-cluster.yaml index 018ba3090..d43fdd37f 100644 --- a/deployment/k8/services/kafka-op/templates/kafka-single-node.yaml +++ b/deployment/k8/services/kafka-op/templates/kafka-cluster.yaml @@ -4,9 +4,9 @@ kind: KafkaNodePool metadata: name: dual-role labels: - strimzi.io/cluster: kafka-cluster + strimzi.io/cluster: {{ .Values.clusterName}} spec: - replicas: 1 + replicas: {{ .Values.nodePool.replicas}} roles: - controller - broker @@ -15,22 +15,22 @@ spec: volumes: - id: 0 type: persistent-claim - size: 10Gi - deleteClaim: true + size: {{ .Values.nodePool.volume.size}} + deleteClaim: {{ .Values.nodePool.volume.deleteClaim}} kraftMetadata: shared --- apiVersion: kafka.strimzi.io/v1beta2 kind: Kafka metadata: - name: kafka-cluster + name: {{ .Values.clusterName}} annotations: strimzi.io/node-pools: enabled strimzi.io/kraft: enabled spec: kafka: - version: 3.8.0 - metadataVersion: 3.8-IV0 + version: {{ .Values.kafkaVersion}} + metadataVersion: {{ .Values.kafkaMetadataVersion}} listeners: - name: plain port: 9092 diff --git a/deployment/k8/services/kafka-op/values.yaml b/deployment/k8/services/kafka-op/values.yaml index 01cce5cc2..e871f0dbf 100644 --- a/deployment/k8/services/kafka-op/values.yaml +++ b/deployment/k8/services/kafka-op/values.yaml @@ -1,38 +1 @@ # Configuration for Strimzi Kafka Operator -#strimzi-kafka-operator: -# You can add operator-specific configurations here if needed -# For example, enabling metrics, RBAC settings, etc. - -## Configuration for the Kafka Cluster -#kafka: -# name: kafka-cluster -# version: "3.6.0" # Kafka version supporting KRaft -# replicas: 1 -# config: -# processRoles: "broker,controller" -# nodeId: 1 -# controllerQuorumVoters: "1@kafka-cluster-0.kafka-cluster-bootstrap:9093" -# interBrokerProtocolVersion: "3.5" -# logMessageFormatVersion: "3.5" -# autoCreateTopicsEnable: "true" -# storage: -# type: "ephemeral" # Use "persistent-claim" for persistent storage -# size: "20Gi" -# class: "managed-nfs-storage" # Set to empty string "" if not using a specific storage class -# deleteClaim: false -# -## Configuration for the Controller -#controller: -# replicas: 1 -# resources: -# limits: -# memory: "2Gi" -# cpu: "1000m" -# requests: -# memory: "1Gi" -# cpu: "500m" -# storage: -# type: "ephemeral" # Use "persistent-claim" for persistent storage -# size: "10Gi" -# class: "managed-nfs-storage" # Set to empty string "" if not using a specific storage class -# deleteClaim: false diff --git a/deployment/k8/services/migration-console/values.yaml b/deployment/k8/services/migration-console/values.yaml index e69b3fc13..3f45c058a 100644 --- a/deployment/k8/services/migration-console/values.yaml +++ b/deployment/k8/services/migration-console/values.yaml @@ -1,3 +1 @@ # Default values for migration-console. - -enabled: false diff --git a/deployment/k8/services/opensearch-helm/Chart.lock b/deployment/k8/services/opensearch-helm/Chart.lock deleted file mode 100644 index 54483035e..000000000 --- a/deployment/k8/services/opensearch-helm/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: opensearch - repository: https://opensearch-project.github.io/helm-charts/ - version: 2.23.1 -digest: sha256:0aab035ab026b655b42c6441cf5ce4e7d05cdec3a37bd049acf1781ab8894514 -generated: "2024-10-11T18:07:51.973102-05:00" diff --git a/deployment/k8/services/opensearch-helm/Chart.yaml b/deployment/k8/services/opensearch-helm/Chart.yaml deleted file mode 100644 index f2f9b6f96..000000000 --- a/deployment/k8/services/opensearch-helm/Chart.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v2 -name: opensearch-helm -version: 0.1.0 -dependencies: - - name: opensearch - version: "2.23.1" - repository: "https://opensearch-project.github.io/helm-charts/" - alias: opensearch-local diff --git a/deployment/k8/services/opensearch-helm/values.yaml b/deployment/k8/services/opensearch-helm/values.yaml deleted file mode 100644 index e79669d6f..000000000 --- a/deployment/k8/services/opensearch-helm/values.yaml +++ /dev/null @@ -1,17 +0,0 @@ -opensearch-local: - fullnameOverride: "target-cluster" - image: - tag: "2.17.0" - replicas: 1 - singleNode: true - config: - discovery.type: single-node - extraEnvs: - - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD - value: myStrongPassword123! -# - name: http.port -# value: "29200" - service: - type: LoadBalancer -# httpPort: 29200 -# httpHostPort: 29200 diff --git a/deployment/k8/services/prometheus/Chart.lock b/deployment/k8/services/prometheus/Chart.lock deleted file mode 100644 index 3dae47c0c..000000000 --- a/deployment/k8/services/prometheus/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: prometheus - repository: https://prometheus-community.github.io/helm-charts - version: 25.27.0 -digest: sha256:30e4a566c45c3dd4ce80a501e9c4d0903d1364a2810aaba965e8796a703a2093 -generated: "2024-10-17T23:31:30.225286-05:00" diff --git a/deployment/k8/services/prometheus/Chart.yaml b/deployment/k8/services/prometheus/Chart.yaml deleted file mode 100644 index 7a4fa048e..000000000 --- a/deployment/k8/services/prometheus/Chart.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v2 -name: prometheus -description: A Helm chart for deploying Prometheus -appVersion: "3.5.0" -version: 0.1.0 -dependencies: - - name: prometheus - version: "25.27.0" - repository: "https://prometheus-community.github.io/helm-charts" diff --git a/deployment/k8/services/prometheus/values.yaml b/deployment/k8/services/prometheus/values.yaml deleted file mode 100644 index 208967dd3..000000000 --- a/deployment/k8/services/prometheus/values.yaml +++ /dev/null @@ -1 +0,0 @@ -#prometheus: