diff --git a/charts/newrelic-logging/templates/configcreator.yaml b/charts/newrelic-logging/templates/configcreator.yaml new file mode 100644 index 000000000..c4b444bfb --- /dev/null +++ b/charts/newrelic-logging/templates/configcreator.yaml @@ -0,0 +1,67 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: newrelic-logging-config-creator + # TODO: do we need namespace and labels like in configmap.yml?annotations: + annotations: + # This makes it so that this job is run first before any other resources + # are created during chart installation/upgrade + "helm.sh/hook": post-install,post-upgrade +spec: + template: + spec: + containers: + - name: config-creator + image: node # TODO: what image should we use + command: + - node + - -e + - | + const fs = require('fs'); + + // Read configuration information + const mappingFileName = '/etc/namespace-to-api-key-mapping/fluent-bit-namespace-to-api-key-mapping.json'; + const mappingFileContents = fs.readFileSync(mappingFileName); + const keysByNamespace = JSON.parse(mappingFileContents); + + // Write out the Fluent Bit config file + const outputBlock = (namespace, key) => { + return `[OUTPUT] + Name newrelic + Match_regex .*_${namespace}_.* + apiKey ${key} + endpoint \${ENDPOINT}`; + }; + const namespaces = Object.keys(keysByNamespace); + const regexTerms = namespaces + .map(namespace => `_${namespace}_`) + .join('|'); + const fallbackRegex = `^(?:(?!${regexTerms}).)*$` + const fallbackBlock = `[OUTPUT] + Name newrelic + Match_regex ${fallbackRegex} + apiKey \${LICENSE_KEY} + endpoint \${ENDPOINT}`; + const teamOutputBlocks = Object.entries(keysByNamespace) + .map(([namespace, key]) => outputBlock(namespace, key)); + const outputBlocks = [ + ...teamOutputBlocks, + fallbackBlock, + ]; + const configFileContents = outputBlocks.join('\n\n'); + const configFileName = '/etc/fluent-bit-teams-config/newrelic-teams-output.conf'; + fs.writeFileSync(configFileName, configFileContents); + volumeMounts: + - name: fluent-bit-namespace-to-api-key-mapping-volume + mountPath: /etc/namespace-to-api-key-mapping + - name: fluent-bit-teams-config-volume + mountPath: /etc/fluent-bit-teams-config + volumes: + - name: fluent-bit-namespace-to-api-key-mapping-volume + configMap: + name: fluent-bit-namespace-to-api-key-mapping + - name: fluent-bit-teams-config-volume + hostPath: + path: /etc/teams-config + # TODO: docs had this, but do we need it? https://docs.openshift.com/dedicated/3/dev_guide/configmaps.html + restartPolicy: Never diff --git a/charts/newrelic-logging/templates/configmap.yaml b/charts/newrelic-logging/templates/configmap.yaml index a92c16fa5..8f7aba489 100644 --- a/charts/newrelic-logging/templates/configmap.yaml +++ b/charts/newrelic-logging/templates/configmap.yaml @@ -6,29 +6,42 @@ metadata: name: {{ template "newrelic-logging.fluentBitConfig" . }} data: fluent-bit.conf: | - {{- if .Values.fluentBit.config.service }} - {{- .Values.fluentBit.config.service | nindent 4 }} - {{- end }} - {{- if .Values.fluentBit.config.inputs }} - {{- .Values.fluentBit.config.inputs | nindent 4 }} - {{- end }} - {{- if .Values.fluentBit.config.extraInputs }} - {{- .Values.fluentBit.config.extraInputs | nindent 4}} - {{- end }} - {{- if and (include "newrelic-logging.lowDataMode" .) (.Values.fluentBit.config.lowDataModeFilters) }} - {{- .Values.fluentBit.config.lowDataModeFilters | nindent 4 }} - {{- else }} - {{- .Values.fluentBit.config.filters | nindent 4 }} - {{- end }} - {{- if .Values.fluentBit.config.extraFilters }} - {{- .Values.fluentBit.config.extraFilters | nindent 4}} - {{- end }} - {{- if .Values.fluentBit.config.outputs }} - {{- .Values.fluentBit.config.outputs | nindent 4 }} - {{- end }} - {{- if .Values.fluentBit.config.extraOutputs }} - {{- .Values.fluentBit.config.extraOutputs | nindent 4}} - {{- end }} + [SERVICE] + Flush 1 + Log_Level ${LOG_LEVEL} + Daemon off + Parsers_File parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + + [INPUT] + Name tail + Tag kube.* + Path ${PATH} + multiline.parser ${LOG_PARSER} + DB ${FB_DB} + Mem_Buf_Limit 7MB + Skip_Long_Lines On + Refresh_Interval 10 + + [FILTER] + Name kubernetes + Match kube.* + # We need the full DNS suffix as Windows only supports resolving names with this suffix + # See: https://kubernetes.io/docs/setup/production-environment/windows/intro-windows-in-kubernetes/#dns-limitations + Kube_URL https://kubernetes.default.svc.cluster.local:443 + Buffer_Size ${K8S_BUFFER_SIZE} + K8S-Logging.Exclude ${K8S_LOGGING_EXCLUDE} + + [FILTER] + Name record_modifier + Match * + Record cluster_name ${CLUSTER_NAME} + + # Include the output blocks that we build during Helm chart installation/upgrade + @INCLUDE /etc/teams-config/newrelic-teams-output.conf + parsers.conf: | {{- if .Values.fluentBit.config.parsers }} {{- .Values.fluentBit.config.parsers | nindent 4}} diff --git a/charts/newrelic-logging/templates/daemonset.yaml b/charts/newrelic-logging/templates/daemonset.yaml index 64bd7b448..2fed45178 100644 --- a/charts/newrelic-logging/templates/daemonset.yaml +++ b/charts/newrelic-logging/templates/daemonset.yaml @@ -139,6 +139,8 @@ spec: volumeMounts: - name: fluent-bit-config mountPath: /fluent-bit/etc + - name: fluent-bit-teams-config + mountPath: /etc/teams-config - name: logs # We mount /var by default because container logs could be on /var/log or /var/lib/docker/containers (symlinked to /var/log) mountPath: {{ .Values.fluentBit.linuxMountPath | default "/var" }} @@ -163,6 +165,9 @@ spec: - name: fluent-bit-config configMap: name: {{ template "newrelic-logging.fluentBitConfig" . }} + - name: fluent-bit-teams-config + hostPath: + path: /etc/teams-config - name: logs hostPath: path: {{ .Values.fluentBit.linuxMountPath | default "/var" }} diff --git a/charts/newrelic-logging/templates/fluent-bit-namespace-to-api-key-mapping.yml b/charts/newrelic-logging/templates/fluent-bit-namespace-to-api-key-mapping.yml new file mode 100644 index 000000000..712e28e63 --- /dev/null +++ b/charts/newrelic-logging/templates/fluent-bit-namespace-to-api-key-mapping.yml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluent-bit-namespace-to-api-key-mapping + # TODO: do we need namespace and labels? I stole them from configmap.yml + namespace: {{ .Release.Namespace }} + labels: {{ include "newrelic-logging.labels" . | indent 4 }} +data: + # TODO: this needs to be a Vault path instead of hard-coding keys + fluent-bit-namespace-to-api-key-mapping.json: | + { + "logging": "" + }