diff --git a/charts/kafka-ha/Chart.yaml b/charts/kafka-ha/Chart.yaml index 8364c8f..3ec2083 100644 --- a/charts/kafka-ha/Chart.yaml +++ b/charts/kafka-ha/Chart.yaml @@ -3,7 +3,7 @@ name: kafka-ha description: Helm chart for Apache Kafka. type: application -version: 15.0.0 +version: 16.0.0 appVersion: v3.6.1 maintainers: diff --git a/charts/kafka-ha/templates/_container.tpl b/charts/kafka-ha/templates/_container.tpl deleted file mode 100644 index 727a1bd..0000000 --- a/charts/kafka-ha/templates/_container.tpl +++ /dev/null @@ -1,35 +0,0 @@ -{{- define "kafka.initContainer.checkClusterId" -}} -- name: check-clusterid - image: {{ include "kafka.kafkaImage" . | quote }} - {{- if .Values.image.pullPolicy }} - imagePullPolicy: {{ .Values.image.pullPolicy | quote }} - {{- end }} - env: - - name: KAFKA_CLUSTER_ID - valueFrom: - secretKeyRef: - name: {{ include "kafka.clusterId.SecretName" . }} - key: clusterId - command: ["/bin/bash"] - args: - - -c - - | - if [[ -f "$KAFKA_CFG_LOG_DIR/meta.properties" ]]; then - meta_clusterid=$(grep -E '^cluster\.id' meta.properties | awk -F '=' '{print $2}') - if [[ "$meta_clusterid" != "$KAFKA_CLUSTER_ID" ]]; then - cat "$KAFKA_CFG_LOG_DIR/meta.properties" - echo "[ERROR] CLUSTER_ID Exception, \ - The CLUSTER_ID currently deployed is $KAFKA_CLUSTER_ID, \ - and The stored CLUSTER_ID in KAFKA_CFG_LOG_DIR is $meta_clusterid" - echo "[ERROR] CLUSTER_ID Exception, \ - Use \"--set clusterId=$meta_clusterid\" to continue helm deploy, \ - Or clean up KAFKA_CFG_LOG_DIR and deploy a new cluster. \ - See https://github.com/sir5kong/kafka-docker" - exit "500" - fi - fi - volumeMounts: - - mountPath: /opt/kafka/data - name: data - readOnly: true -{{- end }} diff --git a/charts/kafka-ha/templates/_helpers.tpl b/charts/kafka-ha/templates/_helpers.tpl index 2b6ee2e..5de1b84 100644 --- a/charts/kafka-ha/templates/_helpers.tpl +++ b/charts/kafka-ha/templates/_helpers.tpl @@ -128,3 +128,10 @@ kafka combinedMode {{- print "false" -}} {{- end -}} {{- end -}} + +{{/* +kafka clusterDomain +*/}} +{{- define "kafka.clusterDomain" -}} +{{- default "cluster.local" .Values.clusterDomain -}} +{{- end -}} diff --git a/charts/kafka-ha/templates/_kafka-env.tpl b/charts/kafka-ha/templates/_kafka-env.tpl index 92b7bd4..8b0510b 100644 --- a/charts/kafka-ha/templates/_kafka-env.tpl +++ b/charts/kafka-ha/templates/_kafka-env.tpl @@ -1,8 +1,17 @@ +{{/* +KAFKA broker domainSuffix +*/}} +{{- define "kafka.broker.domainSuffix" -}} +{{- $serviceName := include "kafka.broker.headless.serviceName" . -}} +{{- $namespace := .Release.Namespace -}} +{{- $clusterDomain := ( include "kafka.clusterDomain" .) -}} +{{- printf "%s.%s.svc.%s" $serviceName $namespace $clusterDomain -}} +{{- end -}} + {{/* KAFKA_CFG_CONTROLLER_QUORUM_VOTERS */}} {{- define "kafka.controller.quorum.voters" -}} -{{- $controllerPort := int .Values.containerPort.controller }} {{- $controllerReplicaCount := int .Values.controller.replicaCount }} {{- $controllerFullName := include "kafka.controller.fullname" . }} {{- $serviceName := include "kafka.controller.headless.serviceName" . }} @@ -12,10 +21,15 @@ KAFKA_CFG_CONTROLLER_QUORUM_VOTERS {{- $serviceName = include "kafka.broker.headless.serviceName" . }} {{- end }} {{- $namespace := .Release.Namespace -}} -{{- range $i := until $controllerReplicaCount }} - {{- if gt $i 0 }}{{ printf "," }}{{ end }}{{ printf "%d@%s-%d.%s.%s.svc:%d" $i $controllerFullName $i $serviceName $namespace $controllerPort }} -{{- end }} -{{- end }} +{{- $clusterDomain := ( include "kafka.clusterDomain" .) -}} +{{- $port := int .Values.containerPort.controller }} +{{- $suffix := printf "%s.%s.svc.%s:%d" $serviceName $namespace $clusterDomain $port -}} + {{- $servers := list -}} + {{- range $i := until $controllerReplicaCount -}} + {{- $servers = printf "%d@%s-%d.%s" $i $controllerFullName $i $suffix | append $servers -}} + {{- end -}} +{{ join "," $servers }} +{{- end -}} {{/* KAFKA Broker Componet label @@ -62,7 +76,6 @@ controller env broker env */}} {{- define "kafka.broker.containerEnv" -}} -{{- $replicaCount := .Values.broker.replicaCount | int -}} - name: POD_HOST_IP valueFrom: fieldRef: @@ -90,7 +103,8 @@ broker env value: "BROKER://0.0.0.0:{{ .Values.containerPort.broker }},EXTERNAL://0.0.0.0:{{ .Values.containerPort.brokerExternal }}" {{- end }} - name: KAFKA_CFG_ADVERTISED_LISTENERS - value: "BROKER://$(POD_IP):{{ .Values.containerPort.broker }}" + {{- $domainSuffix := (include "kafka.broker.domainSuffix" .) }} + value: "BROKER://$(POD_NAME).{{ $domainSuffix }}:{{ .Values.containerPort.broker }}" {{- if .Values.broker.external.enabled }} - name: KAFKA_EXTERNAL_SERVICE_TYPE value: {{ .Values.broker.external.service.type | quote }} @@ -105,13 +119,14 @@ broker env value: CONTROLLER - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS value: {{ include "kafka.controller.quorum.voters" . }} +{{- $replicaCount := .Values.broker.replicaCount | int }} {{- if and $replicaCount (ge $replicaCount 3) }} - name: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR value: "3" - name: KAFKA_CFG_MIN_INSYNC_REPLICAS value: "2" - name: KAFKA_CFG_NUM_PARTITIONS - value: "9" + value: "6" - name: KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR value: "3" - name: KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR @@ -179,15 +194,14 @@ kafka fullNodePorts KAFKA BOOTSTRAPSERVERS */}} {{- define "kafka.bootstrapServers" -}} -{{- $servers := list -}} -{{- $brokerPort := .Values.containerPort.broker | int -}} -{{- $brokerReplicaCount := int .Values.broker.replicaCount -}} {{- $brokerFullName := include "kafka.broker.fullname" . -}} -{{- $serviceName := include "kafka.broker.headless.serviceName" . -}} -{{- $namespace := .Release.Namespace -}} -{{- range $i := until $brokerReplicaCount -}} - {{- $servers = printf "%s-%d.%s.%s.svc:%d" $brokerFullName $i $serviceName $namespace $brokerPort | append $servers -}} -{{- end -}} +{{- $domainSuffix := (include "kafka.broker.domainSuffix" .) -}} +{{- $brokerPort := .Values.containerPort.broker | int -}} + {{- $servers := list -}} + {{- $brokerReplicaCount := int .Values.broker.replicaCount -}} + {{- range $i := until $brokerReplicaCount -}} + {{- $servers = printf "%s-%d.%s:%d" $brokerFullName $i $domainSuffix $brokerPort | append $servers -}} + {{- end -}} {{ join "," $servers }} {{- end -}} diff --git a/charts/kafka-ha/templates/broker/statefulset.yaml b/charts/kafka-ha/templates/broker/statefulset.yaml index c9a22cd..98de880 100644 --- a/charts/kafka-ha/templates/broker/statefulset.yaml +++ b/charts/kafka-ha/templates/broker/statefulset.yaml @@ -92,17 +92,10 @@ spec: {{- end }} env: {{- include "kafka.broker.containerEnv" . | nindent 8 }} - #- name: KAFKA_CFG_ADVERTISED_LISTENERS - # value: "foo" ports: {{- include "kafka.broker.containerPorts" . | nindent 8 }} - livenessProbe: - exec: - command: - - sh - - -c - - bin/kafka-broker-api-versions.sh --bootstrap-server=127.0.0.1:{{ $brokerPort }} {{- with $thisValues.livenessProbe }} + livenessProbe: {{- toYaml . | nindent 10 }} {{- end }} readinessProbe: @@ -132,8 +125,6 @@ spec: preStop: exec: command: ["sh", "-c", "sleep 10; bin/kafka-server-stop.sh"] - initContainers: - {{- include "kafka.initContainer.checkClusterId" . | nindent 8 }} terminationGracePeriodSeconds: {{ $thisValues.terminationGracePeriodSeconds }} volumes: - name: entrypoint-sh diff --git a/charts/kafka-ha/templates/broker/svc-headless.yaml b/charts/kafka-ha/templates/broker/svc-headless.yaml index 5b55f62..2a41741 100644 --- a/charts/kafka-ha/templates/broker/svc-headless.yaml +++ b/charts/kafka-ha/templates/broker/svc-headless.yaml @@ -11,15 +11,13 @@ metadata: spec: type: ClusterIP clusterIP: None - {{- if .Values.broker.combinedMode.enabled }} publishNotReadyAddresses: true - {{- end }} ports: - port: {{ .Values.broker.service.port }} targetPort: broker protocol: TCP name: broker - {{- if .Values.broker.combinedMode.enabled }} + {{- if (eq (include "kafka.combinedMode" .) "true") }} - port: {{ .Values.controller.service.port }} targetPort: controller protocol: TCP diff --git a/charts/kafka-ha/templates/broker/svc.yaml b/charts/kafka-ha/templates/broker/svc.yaml index a086579..0ba0ba1 100644 --- a/charts/kafka-ha/templates/broker/svc.yaml +++ b/charts/kafka-ha/templates/broker/svc.yaml @@ -8,6 +8,9 @@ metadata: labels: {{- include "kafka.labels" . | nindent 4 }} component: {{ $componet | quote }} + {{- with $svcValues.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with $svcValues.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/charts/kafka-ha/templates/controller/statefulset.yaml b/charts/kafka-ha/templates/controller/statefulset.yaml index 54a7349..a31ae48 100644 --- a/charts/kafka-ha/templates/controller/statefulset.yaml +++ b/charts/kafka-ha/templates/controller/statefulset.yaml @@ -126,8 +126,6 @@ spec: preStop: exec: command: ["sh", "-c", "sleep 10; bin/kafka-server-stop.sh"] - initContainers: - {{- include "kafka.initContainer.checkClusterId" . | nindent 8 }} terminationGracePeriodSeconds: {{ $thisValues.terminationGracePeriodSeconds }} volumes: - name: entrypoint-sh diff --git a/charts/kafka-ha/templates/controller/svc.yaml b/charts/kafka-ha/templates/controller/svc.yaml index 5a23461..14ff7c0 100644 --- a/charts/kafka-ha/templates/controller/svc.yaml +++ b/charts/kafka-ha/templates/controller/svc.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.broker.combinedMode.enabled }} +{{- if (eq (include "kafka.combinedMode" .) "false") }} {{- $thisFullName := include "kafka.controller.headless.serviceName" . }} {{- $componet := "controller" }} apiVersion: v1 diff --git a/charts/kafka-ha/templates/exporter/deployment.yaml b/charts/kafka-ha/templates/exporter/deployment.yaml index 897fca4..012276f 100644 --- a/charts/kafka-ha/templates/exporter/deployment.yaml +++ b/charts/kafka-ha/templates/exporter/deployment.yaml @@ -53,7 +53,7 @@ spec: imagePullPolicy: {{ $thisValues.image.pullPolicy }} ports: - containerPort: 9308 - name: http-metrics + name: {{ $thisValues.containerPortName | default "http-metrics"}} protocol: TCP {{- with $thisValues.resources }} resources: diff --git a/charts/kafka-ha/templates/exporter/service.yaml b/charts/kafka-ha/templates/exporter/service.yaml new file mode 100644 index 0000000..edb35f4 --- /dev/null +++ b/charts/kafka-ha/templates/exporter/service.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.exporter.enabled .Values.exporter.service }} +{{- $svcValues := .Values.exporter.service }} +{{- $componet := "kafka-exporter" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kafka.exporter.fullname" . }} + labels: + {{- include "kafka.labels" . | nindent 4 }} + {{- with $svcValues.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $svcValues.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ports: + - name: {{ $svcValues.portName | default "http-metrics" }} + port: {{ $svcValues.port | default 9308 }} + targetPort: {{ .Values.exporter.containerPortName | default "http-metrics" }} + protocol: TCP + selector: + {{- include "kafka.selectorLabels" . | nindent 4 }} + component: {{ $componet | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/kafka-ha/templates/ui/service.yaml b/charts/kafka-ha/templates/ui/service.yaml index bb432e4..16e0a59 100644 --- a/charts/kafka-ha/templates/ui/service.yaml +++ b/charts/kafka-ha/templates/ui/service.yaml @@ -6,6 +6,9 @@ metadata: name: {{ include "kafka.ui.fullname" . }} labels: {{- include "kafka.labels" . | nindent 4 }} + {{- with $svcValues.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with $svcValues.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index c592186..b5dd6ee 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -4,6 +4,7 @@ nameOverride: "kafka" fullnameOverride: "" +clusterDomain: "cluster.local" # It is recommended to use a random cluster id for each cluster. # cat /proc/sys/kernel/random/uuid | base64 | cut -b 1-22 @@ -62,17 +63,13 @@ broker: ## broker.readinessProbe readinessProbe: - initialDelaySeconds: 25 - periodSeconds: 10 + initialDelaySeconds: 50 + periodSeconds: 20 timeoutSeconds: 5 failureThreshold: 3 ## broker.livenessProbe - livenessProbe: - initialDelaySeconds: 300 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 5 + livenessProbe: {} ## broker.nodeSelector nodeSelector: {} @@ -146,13 +143,13 @@ controller: failureThreshold: 3 ## controller.livenessProbe - livenessProbe: - tcpSocket: - port: controller - initialDelaySeconds: 300 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 5 + livenessProbe: {} + # tcpSocket: + # port: controller + # initialDelaySeconds: 300 + # periodSeconds: 30 + # timeoutSeconds: 5 + # failureThreshold: 5 ## controller.nodeSelector nodeSelector: {} @@ -196,6 +193,10 @@ exporter: requests: cpu: 50m memory: 128Mi + + ## exporter.service + service: + annotations: {} ui: enabled: false