From 06ae4f0bef12081788e17904c3d4065623eeb949 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Fri, 6 Sep 2024 14:28:43 -0500 Subject: [PATCH 1/4] wip loki schema hack --- pepr.ts | 4 - src/loki/chart/templates/loki-schema.yaml | 15 +++ src/loki/chart/values.yaml | 1 + src/loki/values/values.yaml | 2 +- src/pepr/loki/README.md | 3 - src/pepr/loki/index.ts | 125 ---------------------- 6 files changed, 17 insertions(+), 133 deletions(-) create mode 100644 src/loki/chart/templates/loki-schema.yaml delete mode 100644 src/pepr/loki/README.md delete mode 100644 src/pepr/loki/index.ts diff --git a/pepr.ts b/pepr.ts index 72ac65826..a8569cd3f 100644 --- a/pepr.ts +++ b/pepr.ts @@ -4,7 +4,6 @@ import { DataStore } from "pepr/dist/lib/storage"; import cfg from "./package.json"; import { istio } from "./src/pepr/istio"; import { Component, setupLogger } from "./src/pepr/logger"; -import { loki } from "./src/pepr/loki"; import { operator } from "./src/pepr/operator"; import { setupAuthserviceSecret } from "./src/pepr/operator/controllers/keycloak/authservice/config"; import { Policy } from "./src/pepr/operator/crd"; @@ -27,9 +26,6 @@ const log = setupLogger(Component.STARTUP); // UDS Core Policies policies, - // Loki schemaConfig update - loki, - // Istio service mesh istio, diff --git a/src/loki/chart/templates/loki-schema.yaml b/src/loki/chart/templates/loki-schema.yaml new file mode 100644 index 000000000..2cf0f279d --- /dev/null +++ b/src/loki/chart/templates/loki-schema.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-schema + namespace: {{ .Release.Namespace }} +data: + # v13schemadate defaults to two days in the future + {{- $configmap := (lookup "v1" "ConfigMap" "loki" "loki-schema") }} + {{- if .Values.v13schemadate }} + v13schemadate: {{ .Values.v13schemadate }} + {{- else if $configmap }} + v13schemadate: {{ $configmap.data.v13schemadate }} + {{- else }} + v13schemadate: {{ (now | dateModify "+48h" | date "2006-01-02") | quote }} + {{- end }} \ No newline at end of file diff --git a/src/loki/chart/values.yaml b/src/loki/chart/values.yaml index e69de29bb..4b45fdbed 100644 --- a/src/loki/chart/values.yaml +++ b/src/loki/chart/values.yaml @@ -0,0 +1 @@ +v13schemadate: "" \ No newline at end of file diff --git a/src/loki/values/values.yaml b/src/loki/values/values.yaml index ac47f79ef..73538441e 100644 --- a/src/loki/values/values.yaml +++ b/src/loki/values/values.yaml @@ -48,7 +48,7 @@ loki: index: prefix: loki_tsdb_ period: 24h - - from: 2024-06-04 + - from: "{{ (lookup \"v1\" \"ConfigMap\" \"loki\" \"loki-schema\").data.v13schemadate }}" store: tsdb object_store: "{{ .Values.loki.storage.type }}" schema: v13 diff --git a/src/pepr/loki/README.md b/src/pepr/loki/README.md deleted file mode 100644 index 5aa0afd32..000000000 --- a/src/pepr/loki/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Loki Pepr Capability - -This project defines a Kubernetes capability using TypeScript and `pepr` to handle mutations for Loki configuration secrets. The primary goal is to manage the `from` date in the `v13` schema configuration of a Loki stack's `config.yaml`. The logic ensures that the date is set only once to a future date during the initial setup and remains unchanged in subsequent updates unless manually altered. diff --git a/src/pepr/loki/index.ts b/src/pepr/loki/index.ts deleted file mode 100644 index b120870b5..000000000 --- a/src/pepr/loki/index.ts +++ /dev/null @@ -1,125 +0,0 @@ -import * as yaml from "js-yaml"; -import { Capability, a } from "pepr"; -import { Component, setupLogger } from "../logger"; - -const log = setupLogger(Component.LOKI); - -export const loki = new Capability({ - name: "loki", - description: "UDS Core Capability for the Loki stack.", -}); - -const { When } = loki; - -// Define the type for the YAML configuration -interface SchemaConfig { - from: string; - store: string; - object_store: string; - schema: string; - index: { - prefix: string; - period: string; - }; -} - -interface LimitsConfig { - allow_structured_metadata: boolean; -} - -interface LokiConfig { - auth_enabled: boolean; - chunk_store_config: object; - common: object; - frontend: object; - frontend_worker: object; - index_gateway: object; - limits_config: LimitsConfig; - memberlist: object; - pattern_ingester: object; - query_range: object; - ruler: object; - runtime_config: object; - schema_config: { - configs: SchemaConfig[]; - }; - server: object; - storage_config: object; - tracing: object; -} - -When(a.Secret) - .IsCreatedOrUpdated() - .WithLabel("app.kubernetes.io/instance", "loki") - .WithLabel("app.kubernetes.io/name", "loki") - .Mutate(async secret => { - log.info( - secret, - `Processing Secret ${secret.Raw.metadata?.namespace}/${secret.Raw.metadata?.name} for Loki schema config date updates.`, - ); - - // Check if the secret contains the "config.yaml" data - if (secret.Raw.data && secret.Raw.data["config.yaml"]) { - let lokiConfig: LokiConfig; - - // Parse the "config.yaml" content into a LokiConfig object - try { - lokiConfig = yaml.load(secret.Raw.data["config.yaml"]) as LokiConfig; - } catch (e) { - log.error(secret, `Failed to parse Loki config.yaml: ${e.message}`); - return; - } - - // Check if the schema_config and its configs array exist - if (lokiConfig.schema_config && Array.isArray(lokiConfig.schema_config.configs)) { - // Find the v13 schema configuration in the array - const v13Config = lokiConfig.schema_config.configs.find(config => config.schema === "v13"); - - if (v13Config) { - // Retrieve the previously stored date from annotations - const storedDate = secret.Raw.metadata?.annotations?.["loki.v13.config.date"]; - const incomingDate = v13Config.from; - - if (!storedDate) { - // If no date is stored, generate a new date 2 days in the future - const currentDate = new Date(); - currentDate.setDate(currentDate.getDate() + 2); - const newDate = currentDate.toISOString().split("T")[0]; // Format as YYYY-MM-DD - - // Update the v13 schema configuration with the new date - v13Config.from = newDate; - - // Ensure limits_config exists and set allow_structured_metadata to false - if (!lokiConfig.limits_config) { - lokiConfig.limits_config = {} as LimitsConfig; - } - lokiConfig.limits_config.allow_structured_metadata = false; - - // Update the secret with the new config.yaml content - secret.Raw.data["config.yaml"] = yaml.dump(lokiConfig); - - // Store the generated date in an annotation for future reference - secret.Raw.metadata!.annotations = { - ...secret.Raw.metadata!.annotations, - "loki.v13.config.date": newDate, - }; - - log.info(`Secret config.yaml updated successfully with new date ${newDate}.`); - } else if (incomingDate === storedDate) { - log.info(`Incoming date matches stored date (${storedDate}). No update needed.`); - } else { - // Log to catch potential manual changes or config drift. - log.warn( - `Incoming date (${incomingDate}) does not match stored date (${storedDate}). No update made.`, - ); - } - } else { - log.error(secret, `v13 schema configuration not found.`); - } - } else { - log.error(secret, `Invalid schema_config or configs in Loki config.yaml.`); - } - } else { - log.error(secret, `No data or config.yaml object found in secret data.`); - } - }); From a72a8a924b22b2581496d17903d1a283313500a2 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Fri, 6 Sep 2024 15:47:43 -0500 Subject: [PATCH 2/4] allow structured metadata false --- src/loki/values/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/loki/values/values.yaml b/src/loki/values/values.yaml index 73538441e..2dfb6f961 100644 --- a/src/loki/values/values.yaml +++ b/src/loki/values/values.yaml @@ -57,6 +57,7 @@ loki: period: 24h limits_config: split_queries_by_interval: "30m" + allow_structured_metadata: false query_scheduler: max_outstanding_requests_per_tenant: 32000 # This is the default in Loki 3.0 extraMemberlistConfig: From 0a8ad993d0b41b7182682e9bc16924d3e909cf34 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Mon, 9 Sep 2024 15:33:11 -0500 Subject: [PATCH 3/4] update images, kick the can --- src/loki/chart/templates/loki-schema.yaml | 15 --------------- src/loki/common/zarf.yaml | 2 +- src/loki/values/registry1-values.yaml | 4 ++-- src/loki/values/unicorn-values.yaml | 4 ++-- src/loki/values/upstream-values.yaml | 4 ++-- src/loki/values/values.yaml | 7 ------- src/loki/zarf.yaml | 12 ++++++------ 7 files changed, 13 insertions(+), 35 deletions(-) delete mode 100644 src/loki/chart/templates/loki-schema.yaml diff --git a/src/loki/chart/templates/loki-schema.yaml b/src/loki/chart/templates/loki-schema.yaml deleted file mode 100644 index 2cf0f279d..000000000 --- a/src/loki/chart/templates/loki-schema.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: loki-schema - namespace: {{ .Release.Namespace }} -data: - # v13schemadate defaults to two days in the future - {{- $configmap := (lookup "v1" "ConfigMap" "loki" "loki-schema") }} - {{- if .Values.v13schemadate }} - v13schemadate: {{ .Values.v13schemadate }} - {{- else if $configmap }} - v13schemadate: {{ $configmap.data.v13schemadate }} - {{- else }} - v13schemadate: {{ (now | dateModify "+48h" | date "2006-01-02") | quote }} - {{- end }} \ No newline at end of file diff --git a/src/loki/common/zarf.yaml b/src/loki/common/zarf.yaml index 84c06af70..3e3901bc4 100644 --- a/src/loki/common/zarf.yaml +++ b/src/loki/common/zarf.yaml @@ -13,7 +13,7 @@ components: localPath: ../chart - name: loki url: https://grafana.github.io/helm-charts/ - version: 6.6.2 + version: 6.12.0 namespace: loki valuesFiles: - ../values/values.yaml diff --git a/src/loki/values/registry1-values.yaml b/src/loki/values/registry1-values.yaml index 8053b90e8..1dc979898 100644 --- a/src/loki/values/registry1-values.yaml +++ b/src/loki/values/registry1-values.yaml @@ -2,7 +2,7 @@ loki: image: registry: registry1.dso.mil repository: ironbank/opensource/grafana/loki - tag: 3.0.0 + tag: 3.1.1 podSecurityContext: fsGroup: 10001 runAsGroup: 10001 @@ -19,7 +19,7 @@ gateway: image: registry: registry1.dso.mil repository: ironbank/opensource/nginx/nginx-alpine - tag: 1.25.3 + tag: 1.26.2 memcached: image: repository: registry1.dso.mil/ironbank/opensource/memcached/memcached diff --git a/src/loki/values/unicorn-values.yaml b/src/loki/values/unicorn-values.yaml index 5a6bffd2d..7de7a9ce3 100644 --- a/src/loki/values/unicorn-values.yaml +++ b/src/loki/values/unicorn-values.yaml @@ -2,12 +2,12 @@ loki: image: registry: cgr.dev repository: du-uds-defenseunicorns/loki - tag: 3.0.0 + tag: 3.1.1 gateway: image: registry: cgr.dev repository: du-uds-defenseunicorns/nginx-fips - tag: 1.25.5 + tag: 1.27.1 memcached: image: repository: cgr.dev/du-uds-defenseunicorns/memcached diff --git a/src/loki/values/upstream-values.yaml b/src/loki/values/upstream-values.yaml index daba7bf7d..e7938fc13 100644 --- a/src/loki/values/upstream-values.yaml +++ b/src/loki/values/upstream-values.yaml @@ -2,13 +2,13 @@ loki: image: registry: docker.io repository: grafana/loki - tag: 3.0.0 + tag: 3.1.1 gateway: image: registry: docker.io repository: nginxinc/nginx-unprivileged - tag: 1.25-alpine + tag: 1.27-alpine memcached: image: diff --git a/src/loki/values/values.yaml b/src/loki/values/values.yaml index 2dfb6f961..5f2d8a019 100644 --- a/src/loki/values/values.yaml +++ b/src/loki/values/values.yaml @@ -48,13 +48,6 @@ loki: index: prefix: loki_tsdb_ period: 24h - - from: "{{ (lookup \"v1\" \"ConfigMap\" \"loki\" \"loki-schema\").data.v13schemadate }}" - store: tsdb - object_store: "{{ .Values.loki.storage.type }}" - schema: v13 - index: - prefix: loki_tsdb_ - period: 24h limits_config: split_queries_by_interval: "30m" allow_structured_metadata: false diff --git a/src/loki/zarf.yaml b/src/loki/zarf.yaml index 180e0e647..64e3d39a2 100644 --- a/src/loki/zarf.yaml +++ b/src/loki/zarf.yaml @@ -16,8 +16,8 @@ components: valuesFiles: - ./values/upstream-values.yaml images: - - docker.io/grafana/loki:3.0.0 - - docker.io/nginxinc/nginx-unprivileged:1.25-alpine + - docker.io/grafana/loki:3.1.1 + - docker.io/nginxinc/nginx-unprivileged:1.27-alpine - docker.io/memcached:1.6.27-alpine - name: loki @@ -32,8 +32,8 @@ components: valuesFiles: - ./values/registry1-values.yaml images: - - registry1.dso.mil/ironbank/opensource/grafana/loki:3.0.0 - - registry1.dso.mil/ironbank/opensource/nginx/nginx-alpine:1.25.3 + - registry1.dso.mil/ironbank/opensource/grafana/loki:3.1.1 + - registry1.dso.mil/ironbank/opensource/nginx/nginx-alpine:1.26.2 - registry1.dso.mil/ironbank/opensource/memcached/memcached:1.6.27 - name: loki @@ -48,6 +48,6 @@ components: valuesFiles: - ./values/unicorn-values.yaml images: - - cgr.dev/du-uds-defenseunicorns/loki:3.0.0 - - cgr.dev/du-uds-defenseunicorns/nginx-fips:1.25.5 + - cgr.dev/du-uds-defenseunicorns/loki:3.1.1 + - cgr.dev/du-uds-defenseunicorns/nginx-fips:1.27.1 - cgr.dev/du-uds-defenseunicorns/memcached:1.6.27 From faa8133bc1f0a6c160e2c16dce6837004c306a76 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Mon, 9 Sep 2024 15:36:54 -0500 Subject: [PATCH 4/4] remove loki config chart values --- src/loki/chart/values.yaml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 src/loki/chart/values.yaml diff --git a/src/loki/chart/values.yaml b/src/loki/chart/values.yaml deleted file mode 100644 index 4b45fdbed..000000000 --- a/src/loki/chart/values.yaml +++ /dev/null @@ -1 +0,0 @@ -v13schemadate: "" \ No newline at end of file