From 492dcc463977e714a80a3614063f5c463bb5bc91 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Tue, 3 Sep 2024 08:12:52 -0600 Subject: [PATCH 01/17] wip: add vector --- src/vector/README.md | 1 + src/vector/chart/.helmignore | 23 ++ src/vector/chart/Chart.yaml | 18 ++ src/vector/chart/templates/_helpers.tpl | 62 ++++ src/vector/chart/templates/uds-exemption.yaml | 23 ++ src/vector/chart/templates/uds-package.yaml | 44 +++ src/vector/chart/values.yaml | 0 src/vector/common/zarf.yaml | 32 +++ src/vector/oscal-component.yaml | 268 ++++++++++++++++++ src/vector/pepr.log | 102 +++++++ src/vector/tasks.yaml | 10 + src/vector/values/registry1-values.yaml | 3 + src/vector/values/unicorn-values.yaml | 3 + src/vector/values/upstream-values.yaml | 3 + src/vector/values/values.yaml | 30 ++ src/vector/zarf.yaml | 48 ++++ 16 files changed, 670 insertions(+) create mode 100644 src/vector/README.md create mode 100644 src/vector/chart/.helmignore create mode 100644 src/vector/chart/Chart.yaml create mode 100644 src/vector/chart/templates/_helpers.tpl create mode 100644 src/vector/chart/templates/uds-exemption.yaml create mode 100644 src/vector/chart/templates/uds-package.yaml create mode 100644 src/vector/chart/values.yaml create mode 100644 src/vector/common/zarf.yaml create mode 100644 src/vector/oscal-component.yaml create mode 100644 src/vector/pepr.log create mode 100644 src/vector/tasks.yaml create mode 100644 src/vector/values/registry1-values.yaml create mode 100644 src/vector/values/unicorn-values.yaml create mode 100644 src/vector/values/upstream-values.yaml create mode 100644 src/vector/values/values.yaml create mode 100644 src/vector/zarf.yaml diff --git a/src/vector/README.md b/src/vector/README.md new file mode 100644 index 000000000..80f646b44 --- /dev/null +++ b/src/vector/README.md @@ -0,0 +1 @@ +## Vector diff --git a/src/vector/chart/.helmignore b/src/vector/chart/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/src/vector/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/src/vector/chart/Chart.yaml b/src/vector/chart/Chart.yaml new file mode 100644 index 000000000..6b5ca4898 --- /dev/null +++ b/src/vector/chart/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: uds-vector-config +description: Vector configuration for UDS + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 diff --git a/src/vector/chart/templates/_helpers.tpl b/src/vector/chart/templates/_helpers.tpl new file mode 100644 index 000000000..7290ba589 --- /dev/null +++ b/src/vector/chart/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "uds-vector-config.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "uds-vector-config.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "uds-vector-config.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "uds-vector-config.labels" -}} +helm.sh/chart: {{ include "uds-vector-config.chart" . }} +{{ include "uds-vector-config.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "uds-vector-config.selectorLabels" -}} +app.kubernetes.io/name: {{ include "uds-vector-config.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "uds-vector-config.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "uds-vector-config.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/src/vector/chart/templates/uds-exemption.yaml b/src/vector/chart/templates/uds-exemption.yaml new file mode 100644 index 000000000..e70a7572a --- /dev/null +++ b/src/vector/chart/templates/uds-exemption.yaml @@ -0,0 +1,23 @@ +apiVersion: uds.dev/v1alpha1 +kind: Exemption +metadata: + name: vector + namespace: uds-policy-exemptions +spec: + exemptions: + - policies: + - DisallowPrivileged + - RequireNonRootUser + - RestrictSELinuxType + - RestrictHostPathWrite + - RestrictVolumeTypes + matcher: + namespace: vector + name: "^vector-.*" + title: "vector exemptions" + description: "Vector mounts the following hostPaths: + - `/var/log/pods`: to tail pod logs + - `/var/lib/docker/containers`: to tail container logs + - `/run/vector`: for Vector's buffering and persistent state + Since logs can have sensitive information, it is better to exclude + Vector from the policy than add the paths as allowable mounts diff --git a/src/vector/chart/templates/uds-package.yaml b/src/vector/chart/templates/uds-package.yaml new file mode 100644 index 000000000..ff39b0ce0 --- /dev/null +++ b/src/vector/chart/templates/uds-package.yaml @@ -0,0 +1,44 @@ +apiVersion: uds.dev/v1alpha1 +kind: Package +metadata: + name: vector + namespace: {{ .Release.Namespace }} +spec: + monitor: + - selector: + app.kubernetes.io/name: vector + targetPort: 3101 + portName: http-metrics + description: Metrics + + network: + allow: + - direction: Ingress + selector: + app.kubernetes.io/name: vector + remoteNamespace: monitoring + remoteSelector: + app.kubernetes.io/name: prometheus + port: 3101 + description: "Prometheus Metrics" + + - direction: Egress + selector: + app.kubernetes.io/name: vector + remoteGenerated: KubeAPI + + - direction: Egress + remoteNamespace: tempo + remoteSelector: + app.kubernetes.io/name: tempo + port: 9411 + description: "Tempo" + + - direction: Egress + selector: + app.kubernetes.io/name: vector + remoteNamespace: loki + remoteSelector: + app.kubernetes.io/name: loki + port: 8080 + description: "Write Logs to Loki" diff --git a/src/vector/chart/values.yaml b/src/vector/chart/values.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/src/vector/common/zarf.yaml b/src/vector/common/zarf.yaml new file mode 100644 index 000000000..118d920b3 --- /dev/null +++ b/src/vector/common/zarf.yaml @@ -0,0 +1,32 @@ +kind: ZarfPackageConfig +metadata: + name: uds-core-vector-common + description: "UDS Core Vector Common" + url: "https://vector.dev/" + +components: + - name: vector + required: true + charts: + - name: uds-vector-config + namespace: vector + version: 0.1.0 + localPath: ../chart + - name: vector + url: https://helm.vector.dev + version: 0.35.0 + namespace: vector + gitPath: charts/vector + valuesFiles: + - ../values/values.yaml + actions: + onDeploy: + after: + - description: Validate Vector Package + maxTotalSeconds: 300 + wait: + cluster: + kind: Packages + name: vector + namespace: vector + condition: "'{.status.phase}'=Ready" diff --git a/src/vector/oscal-component.yaml b/src/vector/oscal-component.yaml new file mode 100644 index 000000000..1a5bb3497 --- /dev/null +++ b/src/vector/oscal-component.yaml @@ -0,0 +1,268 @@ +component-definition: + uuid: ff959bdb-7be9-49b3-9dc2-c41b34e7017d + metadata: + title: Vector + last-modified: "2024-01-31T16:44:35Z" + version: "20240132" + oscal-version: 1.1.2 + parties: + - uuid: f3cf70f8-ba44-4e55-9ea3-389ef24847d3 + type: organization + name: Defense Unicorns + links: + - href: https://defenseunicorns.com + rel: website + components: + - uuid: 3ca1e9a3-a566-48d1-93af-200abd1245e3 + type: software + title: Vector + description: | + Log collector + purpose: Collects logs from the cluster + responsible-roles: + - role-id: provider + party-uuids: + - f3cf70f8-ba44-4e55-9ea3-389ef24847d3 + control-implementations: + - uuid: d2afb4c4-2cd8-5305-a6cc-d1bc7b388d0c + source: https://raw.githubusercontent.com/GSA/fedramp-automation/93ca0e20ff5e54fc04140613476fba80f08e3c7d/dist/content/rev5/baselines/json/FedRAMP_rev5_HIGH-baseline-resolved-profile_catalog.json + description: Controls implemented by Vector for inheritance by applications + implemented-requirements: + - uuid: 954ba9c8-452c-4503-a43f-c880a01b828d + control-id: ac-6.9 + description: >- + # Control Description + Misuse of privileged functions, either intentionally or unintentionally by authorized users, or by unauthorized external entities that have compromised information system accounts, is a serious and ongoing concern and can have significant adverse impacts on organizations. + Auditing the use of privileged functions is one way to detect such misuse, and in doing so, help mitigate the risk from insider threats and the advanced persistent threat (APT). + + # Control Implementation + Vector can be configured to collect all logs from Kubernetes and underlying operating systems, allowing the aggregation of privileged function calls. + remarks: This control is fully implemented by this tool. + links: + - href: "#98b97ec9-a9ce-4444-83d8-71066270a424" + rel: reference + text: Lula Validation + - href: "#fbe5855d-b4ea-4ff5-9f0d-5901d620577a" + rel: reference + text: Lula Validation + + - uuid: 2a25a5a4-4fbc-4fbc-88e3-2e34ddc3fb0e + control-id: au-2 + description: >- + # Control Description + An event is any observable occurrence in an organizational information system. + Organizations identify audit events as those events which are significant and relevant to the security of information systems and the environments in which those systems operate in order to meet specific and ongoing audit needs. + Audit events can include, for example, password changes, failed logons, or failed accesses related to information systems, administrative privilege usage, PIV credential usage, or third-party credential usage. + In determining the set of auditable events, organizations consider the auditing appropriate for each of the security controls to be implemented. + To balance auditing requirements with other information system needs, this control also requires identifying that subset of auditable events that are audited at a given point in time. + + # Control Implementation + Logging daemons are present on each node that BigBang is installed on. Out of the box, the following events are captured: + * all containers emitting to STDOUT or STDERR (captured by container runtime translating container logs to /var/log/containers). + * all kubernetes api server requests. + * all events emitted by the kubelet. + remarks: This control is fully implemented by this tool. + links: + - href: "#98b97ec9-a9ce-4444-83d8-71066270a424" + rel: reference + text: Lula Validation + - href: "#0be7345d-e9d3-4248-9c14-5fed8e7bfa01" + rel: reference + text: Lula Validation + + - uuid: 762604db-77ec-415f-8728-c296873ab48b + control-id: au-3 + description: >- + # Control Description + Audit record content that may be necessary to satisfy the requirement of this control, includes, for example, time stamps, source and destination addresses, user/process identifiers, event descriptions, success/fail indications, filenames involved, and access control or flow control rules invoked. + Event outcomes can include indicators of event success or failure and event-specific results (e.g., the security state of the information system after the event occurred). + + # Control Implementation + Logs are captured by vector from the node. The node logs will contain the necessary log data from all pods/applications inside the selected nodes. + Validating `logfmt` as the config.logFormat would be the goal. This is currently a secret mounted to /etc/vector/vector.yaml in the vector container. We will ensure the vector.yaml file is at a minimum the target config. + https://grafana.com/docs/loki/latest/send-data/vector/stages/logfmt/ + remarks: This control is fully implemented by this tool. + links: + - href: "#98b97ec9-a9ce-4444-83d8-71066270a424" + rel: reference + text: Lula Validation + - href: "#9bfc68e0-381a-4006-9f68-c293e3b20cee" + rel: reference + text: Lula Validation + + - uuid: 9ad7ddfb-4701-4c34-88f7-9d85abb13d60 + control-id: au-8 + description: >- + # Control Description + Time stamps generated by the information system include date and time. + Time is commonly expressed in Coordinated Universal Time (UTC), a modern continuation of Greenwich Mean Time (GMT), or local time with an offset from UTC. + Granularity of time measurements refers to the degree of synchronization between information system clocks and reference clocks, for example, clocks synchronizing within hundreds of milliseconds or within tens of milliseconds. + Organizations may define different time granularities for different system components. + Time service can also be critical to other security capabilities such as access control and identification and authentication, depending on the nature of the mechanisms used to support those capabilities. + + # Control Implementation + Records captured by the logging daemon are enriched to ensure the following are always present: + * time of the event (UTC). + * source of event (pod, namespace, container id). + Applications are responsible for providing all other information. + Validating `logfmt` as the config.logFormat would be the goal. This is currently a secret mounted to /etc/vector/vector.yaml in the vector container. We will ensure the vector.yaml file is at a minimum the target config. + https://grafana.com/docs/loki/latest/send-data/vector/stages/logfmt/ + remarks: This control is fully implemented by this tool. + links: + - href: "#98b97ec9-a9ce-4444-83d8-71066270a424" + rel: reference + text: Lula Validation + - href: "#9bfc68e0-381a-4006-9f68-c293e3b20cee" + rel: reference + text: Lula Validation + props: + - name: framework + ns: https://docs.lula.dev/oscal/ns + value: il4 + back-matter: + resources: + - uuid: D552C935-E40C-4A03-B5CC-4605EBD95B6D + title: Vector + rlinks: + - href: https://grafana.com/docs/loki/latest/clients/vector/ + - uuid: 211C474B-E11A-4DD2-8075-50CDAC507CDC + title: Big Bang Vector package + rlinks: + - href: https://repo1.dso.mil/platform-one/big-bang/apps/sandbox/vector + - uuid: 98b97ec9-a9ce-4444-83d8-71066270a424 + title: Lula Validation + rlinks: + - href: lula.dev + remarks: Validation health check + description: >- + target: + provider: opa + domain: kubernetes + payload: + resources: + - name: daemonsets + resourceRule: + Group: apps + Version: v1 + Resource: daemonsets + Namespaces: [vector] + rego: | + package validate + + import future.keywords.every + + validate { + every daemonset in input.daemonsets { + daemonset.kind == "DaemonSet" + podsScheduled := daemonset.status.desiredNumberScheduled + numberAvailable := daemonset.status.numberAvailable + numberReady := daemonset.status.numberReady + podsScheduled == numberAvailable + numberAvailable == numberReady + } + } + - uuid: fbe5855d-b4ea-4ff5-9f0d-5901d620577a + title: Lula Validation + remarks: Log the execution of privileged functions. + rlinks: + - href: lula.dev + description: >- + target: + provider: opa + domain: kubernetes + payload: + resources: + - name: pods + resourceRule: + Group: + Version: v1 + Resource: pods + Namespaces: [vector] + rego: | + package validate + + import future.keywords.every + + validate { + every pod in input.pods { + volumes := pod.spec.volumes + + some volume in volumes + volume.name == "varlog" + volume.hostPath.path == "/var/log" + } + } + - uuid: 0be7345d-e9d3-4248-9c14-5fed8e7bfa01 + title: Lula Validation + remarks: + a. Identify the types of events that the system is capable of logging in support of the audit function for organization-defined event types that the system is capable of logging; + b. Coordinate the event logging function with other organizational entities requiring audit-related information to guide and inform the selection criteria for events to be logged; + c. Specify the following event types for logging within the system organization-defined event types (subset of the event types defined in AU-2a.) along with the frequency of (or situation requiring) logging for each identified event type; + d. Provide a rationale for why the event types selected for logging are deemed to be adequate to support after-the-fact investigations of incidents; and + e. Review and update the event types selected for logging on an organization-defined frequency. + rlinks: + - href: lula.dev + description: >- + target: + provider: opa + domain: kubernetes + payload: + resources: + - name: pods + resourceRule: + Group: + Version: v1 + Resource: pods + Namespaces: [vector] + rego: | + package validate + + import future.keywords.every + + validate { + every pod in input.pods { + volumes := pod.spec.volumes + + some volume in volumes + volume.name == "pods" + volume.hostPath.path == "/var/log/pods" + } + } + - uuid: 9bfc68e0-381a-4006-9f68-c293e3b20cee + title: Lula Validation + remarks: Ensure that audit records contain information that establishes the following; + a. What type of event occurred; + b. When the event occurred; + c. Where the event occurred; + d. Source of the event; + e. Outcome of the event; and + f. Identity of any individuals, subjects, or objects/entities associated with the event. + rlinks: + - href: lula.dev + description: >- + target: + provider: opa + domain: kubernetes + payload: + resources: + - name: pods + resourceRule: + Group: + Version: v1 + Resource: pods + Namespaces: [vector] + rego: | + package validate + + import future.keywords.every + + validate { + every pod in input.pods { + containers := pod.spec.containers + + some container in containers + container.name == "vector" + some i + container.args[i] == "-config.file=/etc/vector/vector.yaml" + } + } diff --git a/src/vector/pepr.log b/src/vector/pepr.log new file mode 100644 index 000000000..ec479815b --- /dev/null +++ b/src/vector/pepr.log @@ -0,0 +1,102 @@ +watcher {"level":20,"time":1725043340959,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /api/v1/pods."} +watcher {"level":20,"time":1725043340960,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /api/v1/services."} +watcher {"level":20,"time":1725043340960,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /apis/discovery.k8s.io/v1/endpointslices."} +watcher {"level":20,"time":1725043340961,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /apis/uds.dev/v1alpha1/packages."} +watcher {"level":20,"time":1725043340961,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /apis/uds.dev/v1alpha1/packages."} +watcher {"level":20,"time":1725043340963,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Resetting pending promise and dequeuing"} +watcher {"level":20,"time":1725043340963,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Reconciling kubernetes"} +watcher {"level":20,"time":1725043340963,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","component":"operator.generators","msg":"Processing watch for api service, getting endpoint slices for updating API server CIDR"} +watcher {"level":20,"time":1725043340968,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Resetting pending promise and dequeuing"} +watcher {"level":20,"time":1725043340968,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"No element, not dequeuing"} +watcher {"level":20,"time":1725043343189,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","apiVersion":"pepr.dev/v1","data":{"__pepr_do_not_delete__":"k-thx-bye"},"kind":"PeprStore","metadata":{"creationTimestamp":"2024-08-30T18:42:20Z","generation":1,"managedFields":[{"apiVersion":"pepr.dev/v1","fieldsType":"FieldsV1","fieldsV1":{"f:data":{"f:__pepr_do_not_delete__":{}}},"manager":"pepr","operation":"Apply","time":"2024-08-30T18:42:20Z"}],"name":"pepr-uds-core-schedule","namespace":"pepr-system","resourceVersion":"1486","uid":"f1f24a26-532e-4873-b901-bf768f0c16ce"},"msg":"Pepr Store migration"} +watcher {"level":50,"time":1725043343190,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","err":{"type":"Error","message":"No operations specified","stack":"Error: No operations specified\n at Object.Patch (/app/node_modules/kubernetes-fluent-client/dist/fluent/index.js:195:19)\n at flushCache (/app/node_modules/pepr/dist/lib.js:735:101)\n at #migrateAndSetupWatch (/app/node_modules/pepr/dist/lib.js:777:11)\n at /app/node_modules/pepr/dist/lib.js:716:159\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)"},"msg":"Pepr store update failure"} +watcher {"level":20,"time":1725043343198,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","apiVersion":"pepr.dev/v1","data":{"__pepr_do_not_delete__":"k-thx-bye"},"kind":"PeprStore","metadata":{"creationTimestamp":"2024-08-30T18:42:20Z","generation":1,"managedFields":[{"apiVersion":"pepr.dev/v1","fieldsType":"FieldsV1","fieldsV1":{"f:data":{"f:__pepr_do_not_delete__":{}}},"manager":"pepr","operation":"Apply","time":"2024-08-30T18:42:20Z"}],"name":"pepr-uds-core-schedule","namespace":"pepr-system","resourceVersion":"1486","uid":"f1f24a26-532e-4873-b901-bf768f0c16ce"},"msg":"Pepr Store update"} +watcher {"level":30,"time":1725043343198,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"✅ Scheduling processed"} +watcher {"level":30,"time":1725043353257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"2 ms"} +watcher {"level":30,"time":1725043353259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":20,"time":1725043357109,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","apiVersion":"uds.dev/v1alpha1","kind":"Package","metadata":{"annotations":{"meta.helm.sh/release-name":"uds-metrics-server-config","meta.helm.sh/release-namespace":"metrics-server"},"creationTimestamp":"2024-08-30T18:42:37Z","generation":1,"labels":{"app.kubernetes.io/managed-by":"Helm"},"managedFields":[{"apiVersion":"uds.dev/v1alpha1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app.kubernetes.io/managed-by":{}}},"f:spec":{".":{},"f:network":{".":{},"f:allow":{}}}},"manager":"uds","operation":"Update","time":"2024-08-30T18:42:37Z"}],"name":"metrics-server","namespace":"metrics-server","resourceVersion":"1541","uid":"02935348-664a-40f6-a88a-e6cb997e2c52"},"spec":{"network":{"allow":[{"direction":"Egress","port":10250,"remoteGenerated":"Anywhere","selector":{"app.kubernetes.io/name":"metrics-server"}},{"direction":"Egress","remoteGenerated":"KubeAPI","selector":{"app.kubernetes.io/name":"metrics-server"}},{"direction":"Ingress","port":10250,"remoteGenerated":"Anywhere","selector":{"app.kubernetes.io/name":"metrics-server"}}]}},"msg":"Watch event ADDED received"} +watcher {"level":20,"time":1725043357110,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","apiVersion":"uds.dev/v1alpha1","kind":"Package","metadata":{"annotations":{"meta.helm.sh/release-name":"uds-metrics-server-config","meta.helm.sh/release-namespace":"metrics-server"},"creationTimestamp":"2024-08-30T18:42:37Z","generation":1,"labels":{"app.kubernetes.io/managed-by":"Helm"},"managedFields":[{"apiVersion":"uds.dev/v1alpha1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app.kubernetes.io/managed-by":{}}},"f:spec":{".":{},"f:network":{".":{},"f:allow":{}}}},"manager":"uds","operation":"Update","time":"2024-08-30T18:42:37Z"}],"name":"metrics-server","namespace":"metrics-server","resourceVersion":"1541","uid":"02935348-664a-40f6-a88a-e6cb997e2c52"},"spec":{"network":{"allow":[{"direction":"Egress","port":10250,"remoteGenerated":"Anywhere","selector":{"app.kubernetes.io/name":"metrics-server"}},{"direction":"Egress","remoteGenerated":"KubeAPI","selector":{"app.kubernetes.io/name":"metrics-server"}},{"direction":"Ingress","port":10250,"remoteGenerated":"Anywhere","selector":{"app.kubernetes.io/name":"metrics-server"}}]}},"msg":"Watch event ADDED received"} +watcher {"level":20,"time":1725043357110,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Enqueueing metrics-server/metrics-server"} +watcher {"level":20,"time":1725043357110,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Reconciling metrics-server"} +watcher {"level":20,"time":1725043357110,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Resetting pending promise and dequeuing"} +watcher {"level":20,"time":1725043357110,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"No element, not dequeuing"} +watcher {"level":30,"time":1725043363256,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043363258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043373262,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"2 ms"} +watcher {"level":30,"time":1725043373263,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043383257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043383258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043393256,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043393257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043403258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043403259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043413263,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043413265,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043423258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043423258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043433259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043433259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043443259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043443260,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043453258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043453258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043463260,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043463262,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043473259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043473259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043483262,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043483264,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043493258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043493258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043503257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043503258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043513257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043513259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043523258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043523259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043533256,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043533256,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043543258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043543259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043553264,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043553265,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043563258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043563259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043573258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043573258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043583257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043583258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043593261,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043593261,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043603258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043603260,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043613261,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043613262,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043623256,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043623257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043633257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043633259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043643259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043643259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043653260,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043653260,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":20,"time":1725043659051,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event reconnect received. Reconnecting after 1 attempt."} +watcher {"level":20,"time":1725043659060,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event list received. {\n \"apiVersion\": \"uds.dev/v1alpha1\",\n \"items\": [\n {\n \"apiVersion\": \"uds.dev/v1alpha1\",\n \"kind\": \"Package\",\n \"metadata\": {\n \"annotations\": {\n \"meta.helm.sh/release-name\": \"uds-metrics-server-config\",\n \"meta.helm.sh/release-namespace\": \"metrics-server\"\n },\n \"creationTimestamp\": \"2024-08-30T18:42:37Z\",\n \"generation\": 1,\n \"labels\": {\n \"app.kubernetes.io/managed-by\": \"Helm\"\n },\n \"managedFields\": [\n {\n \"apiVersion\": \"uds.dev/v1alpha1\",\n \"fieldsType\": \"FieldsV1\",\n \"fieldsV1\": {\n \"f:metadata\": {\n \"f:annotations\": {\n \".\": {},\n \"f:meta.helm.sh/release-name\": {},\n \"f:meta.helm.sh/release-namespace\": {}\n },\n \"f:labels\": {\n \".\": {},\n \"f:app.kubernetes.io/managed-by\": {}\n }\n },\n \"f:spec\": {\n \".\": {},\n \"f:network\": {\n \".\": {},\n \"f:allow\": {}\n }\n }\n },\n \"manager\": \"uds\",\n \"operation\": \"Update\",\n \"time\": \"2024-08-30T18:42:37Z\"\n }\n ],\n \"name\": \"metrics-server\",\n \"namespace\": \"metrics-server\",\n \"resourceVersion\": \"1541\",\n \"uid\": \"02935348-664a-40f6-a88a-e6cb997e2c52\"\n },\n \"spec\": {\n \"network\": {\n \"allow\": [\n {\n \"direction\": \"Egress\",\n \"port\": 10250,\n \"remoteGenerated\": \"Anywhere\",\n \"selector\": {\n \"app.kubernetes.io/name\": \"metrics-server\"\n }\n },\n {\n \"direction\": \"Egress\",\n \"remoteGenerated\": \"KubeAPI\",\n \"selector\": {\n \"app.kubernetes.io/name\": \"metrics-server\"\n }\n },\n {\n \"direction\": \"Ingress\",\n \"port\": 10250,\n \"remoteGenerated\": \"Anywhere\",\n \"selector\": {\n \"app.kubernetes.io/name\": \"metrics-server\"\n }\n }\n ]\n }\n }\n }\n ],\n \"kind\": \"PackageList\",\n \"metadata\": {\n \"continue\": \"\",\n \"resourceVersion\": \"1886\"\n }\n}."} +watcher {"level":20,"time":1725043659063,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /apis/uds.dev/v1alpha1/packages."} +watcher {"level":20,"time":1725043660574,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event reconnect received. Reconnecting after 1 attempt."} +watcher {"level":20,"time":1725043660582,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event list received. {\n \"apiVersion\": \"uds.dev/v1alpha1\",\n \"items\": [\n {\n \"apiVersion\": \"uds.dev/v1alpha1\",\n \"kind\": \"Package\",\n \"metadata\": {\n \"annotations\": {\n \"meta.helm.sh/release-name\": \"uds-metrics-server-config\",\n \"meta.helm.sh/release-namespace\": \"metrics-server\"\n },\n \"creationTimestamp\": \"2024-08-30T18:42:37Z\",\n \"generation\": 1,\n \"labels\": {\n \"app.kubernetes.io/managed-by\": \"Helm\"\n },\n \"managedFields\": [\n {\n \"apiVersion\": \"uds.dev/v1alpha1\",\n \"fieldsType\": \"FieldsV1\",\n \"fieldsV1\": {\n \"f:metadata\": {\n \"f:annotations\": {\n \".\": {},\n \"f:meta.helm.sh/release-name\": {},\n \"f:meta.helm.sh/release-namespace\": {}\n },\n \"f:labels\": {\n \".\": {},\n \"f:app.kubernetes.io/managed-by\": {}\n }\n },\n \"f:spec\": {\n \".\": {},\n \"f:network\": {\n \".\": {},\n \"f:allow\": {}\n }\n }\n },\n \"manager\": \"uds\",\n \"operation\": \"Update\",\n \"time\": \"2024-08-30T18:42:37Z\"\n }\n ],\n \"name\": \"metrics-server\",\n \"namespace\": \"metrics-server\",\n \"resourceVersion\": \"1541\",\n \"uid\": \"02935348-664a-40f6-a88a-e6cb997e2c52\"\n },\n \"spec\": {\n \"network\": {\n \"allow\": [\n {\n \"direction\": \"Egress\",\n \"port\": 10250,\n \"remoteGenerated\": \"Anywhere\",\n \"selector\": {\n \"app.kubernetes.io/name\": \"metrics-server\"\n }\n },\n {\n \"direction\": \"Egress\",\n \"remoteGenerated\": \"KubeAPI\",\n \"selector\": {\n \"app.kubernetes.io/name\": \"metrics-server\"\n }\n },\n {\n \"direction\": \"Ingress\",\n \"port\": 10250,\n \"remoteGenerated\": \"Anywhere\",\n \"selector\": {\n \"app.kubernetes.io/name\": \"metrics-server\"\n }\n }\n ]\n }\n }\n }\n ],\n \"kind\": \"PackageList\",\n \"metadata\": {\n \"continue\": \"\",\n \"resourceVersion\": \"1890\"\n }\n}."} +watcher {"level":20,"time":1725043660584,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /apis/uds.dev/v1alpha1/packages."} +watcher {"level":30,"time":1725043663258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043663259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043673259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043673259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043683262,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043683265,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} +watcher {"level":30,"time":1725043693258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043693258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043703258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043703258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043713258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043713259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043723259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} +watcher {"level":30,"time":1725043723260,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} diff --git a/src/vector/tasks.yaml b/src/vector/tasks.yaml new file mode 100644 index 000000000..3a71a4295 --- /dev/null +++ b/src/vector/tasks.yaml @@ -0,0 +1,10 @@ +tasks: + - name: validate + actions: + - description: Validate vector + wait: + cluster: + kind: Pod + name: app.kubernetes.io/instance=vector + namespace: vector + condition: Ready diff --git a/src/vector/values/registry1-values.yaml b/src/vector/values/registry1-values.yaml new file mode 100644 index 000000000..9ff27eecd --- /dev/null +++ b/src/vector/values/registry1-values.yaml @@ -0,0 +1,3 @@ +image: + repository: timberio/vector + tag: 0.40.1-distroless-static diff --git a/src/vector/values/unicorn-values.yaml b/src/vector/values/unicorn-values.yaml new file mode 100644 index 000000000..6a129b1e6 --- /dev/null +++ b/src/vector/values/unicorn-values.yaml @@ -0,0 +1,3 @@ +image: + repository: cgr.dev/du-uds-defenseunicorns/vector + tag: 0.40.1 diff --git a/src/vector/values/upstream-values.yaml b/src/vector/values/upstream-values.yaml new file mode 100644 index 000000000..9ff27eecd --- /dev/null +++ b/src/vector/values/upstream-values.yaml @@ -0,0 +1,3 @@ +image: + repository: timberio/vector + tag: 0.40.1-distroless-static diff --git a/src/vector/values/values.yaml b/src/vector/values/values.yaml new file mode 100644 index 000000000..c0131e0c5 --- /dev/null +++ b/src/vector/values/values.yaml @@ -0,0 +1,30 @@ +# Run as an agent daemonset +role: "Agent" + +# todo: all this +customConfig: + data_dir: /vector-data-dir + api: + enabled: true + address: 127.0.0.1:8686 + playground: false + sources: + vector: + address: 0.0.0.0:6000 + type: vector + version: "2" + sinks: + stdout: + type: console + inputs: [vector] + encoding: + codec: json + +persistence: + enabled: true + hostPath: + enabled: true + path: "/var/lib/vector" + +podMonitor: + enabled: true diff --git a/src/vector/zarf.yaml b/src/vector/zarf.yaml new file mode 100644 index 000000000..b661d6c01 --- /dev/null +++ b/src/vector/zarf.yaml @@ -0,0 +1,48 @@ +kind: ZarfPackageConfig +metadata: + name: uds-core-vector + description: "UDS Core Vector" + url: "https://vector.dev/" + +components: + - name: vector + required: true + description: "Deploy Vector" + only: + flavor: upstream + import: + path: common + charts: + - name: vector + valuesFiles: + - values/upstream-values.yaml + images: + - timberio/vector:0.40.1-distroless-static + + - name: vector + required: true + description: "Deploy Vector" + only: + flavor: registry1 + import: + path: common + charts: + - name: vector + valuesFiles: + - values/registry1-values.yaml + images: + - timberio/vector:0.40.1-distroless-static # registry1 image is WIP + + - name: vector + required: true + description: "Deploy Vector" + only: + flavor: unicorn + import: + path: common + charts: + - name: vector + valuesFiles: + - values/unicorn-values.yaml + images: + - cgr.dev/du-uds-defenseunicorns/vector:0.40.1 From 9f0ac4a6b854e9bc9c16356e195d13f8fa995e8f Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 4 Sep 2024 11:26:06 -0600 Subject: [PATCH 02/17] chore: vector config with labelling >> loki --- .github/filters.yaml | 4 +- .vscode/settings.json | 4 +- README.md | 2 +- compliance/oscal-component.yaml | 2 +- docs/application-baseline.md | 2 +- .../resource-configuration-and-ha.md | 14 +- packages/standard/zarf.yaml | 6 +- renovate.json | 6 +- src/loki/chart/templates/uds-package.yaml | 6 +- .../controllers/exemptions/exemptions.spec.ts | 38 +-- src/pepr/policies/exemptions/index.spec.ts | 2 +- src/promtail/README.md | 1 - src/promtail/chart/.helmignore | 23 -- src/promtail/chart/Chart.yaml | 18 -- src/promtail/chart/templates/_helpers.tpl | 62 ---- src/promtail/chart/templates/service.yaml | 18 -- .../chart/templates/uds-exemption.yaml | 24 -- src/promtail/chart/templates/uds-package.yaml | 44 --- src/promtail/chart/values.yaml | 0 src/promtail/common/zarf.yaml | 32 --- src/promtail/oscal-component.yaml | 268 ------------------ src/promtail/tasks.yaml | 10 - src/promtail/values/registry1-values.yaml | 10 - src/promtail/values/unicorn-values.yaml | 10 - src/promtail/values/upstream-values.yaml | 10 - src/promtail/values/values.yaml | 116 -------- src/promtail/zarf.yaml | 51 ---- src/vector/chart/templates/uds-exemption.yaml | 8 +- src/vector/chart/templates/uds-package.yaml | 23 -- src/vector/common/zarf.yaml | 6 + src/vector/values/values.yaml | 105 ++++++- 31 files changed, 147 insertions(+), 778 deletions(-) delete mode 100644 src/promtail/README.md delete mode 100644 src/promtail/chart/.helmignore delete mode 100644 src/promtail/chart/Chart.yaml delete mode 100644 src/promtail/chart/templates/_helpers.tpl delete mode 100644 src/promtail/chart/templates/service.yaml delete mode 100644 src/promtail/chart/templates/uds-exemption.yaml delete mode 100644 src/promtail/chart/templates/uds-package.yaml delete mode 100644 src/promtail/chart/values.yaml delete mode 100644 src/promtail/common/zarf.yaml delete mode 100644 src/promtail/oscal-component.yaml delete mode 100644 src/promtail/tasks.yaml delete mode 100644 src/promtail/values/registry1-values.yaml delete mode 100644 src/promtail/values/unicorn-values.yaml delete mode 100644 src/promtail/values/upstream-values.yaml delete mode 100644 src/promtail/values/values.yaml delete mode 100644 src/promtail/zarf.yaml diff --git a/.github/filters.yaml b/.github/filters.yaml index 29fbf82ca..77ec1bba8 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -73,8 +73,8 @@ prometheus-stack: - "!**/*.gif" - "!**/*.svg" -promtail: - - "src/promtail/**" +vector: + - "src/vector/**" - "!**/*.md" - "!**/*.jpg" - "!**/*.png" diff --git a/.vscode/settings.json b/.vscode/settings.json index ed63700cc..10870e798 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,7 +12,6 @@ "https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.2/uds.schema.json": [ "uds-bundle.yaml" ], - // renovate: datasource=github-tags depName=defenseunicorns/uds-cli versioning=semver "https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.2/tasks.schema.json": [ "tasks.yaml", @@ -37,7 +36,6 @@ "MITM", "neuvector", "opensource", - "promtail", "Quarkus", "Quickstart", "seccomp", @@ -47,7 +45,7 @@ "cSpell.enabled": true, "[typescript]": { "editor.codeActionsOnSave": { - "source.organizeImports": "always" + "source.organizeImports": "always" } }, } diff --git a/README.md b/README.md index 3096ffe19..d444ae1f6 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ UDS Core establishes a secure baseline for cloud-native systems and ships with c - [Neuvector](https://open-docs.neuvector.com/) - Container Security - [Pepr](https://pepr.dev) - UDS policy engine & operator - [Prometheus Stack](https://github.com/prometheus-operator/kube-prometheus) - Monitoring -- [Promtail](https://grafana.com/docs/loki/latest/send-data/promtail/) - Log Aggregation +- [Vector](https://vector.dev/) - Log Aggregation - [Velero](https://velero.io/) - Backup & Restore #### Future Applications diff --git a/compliance/oscal-component.yaml b/compliance/oscal-component.yaml index ecb88933e..4be69f019 100644 --- a/compliance/oscal-component.yaml +++ b/compliance/oscal-component.yaml @@ -19,7 +19,7 @@ component-definition: - href: 'file://./../src/loki/oscal-component.yaml' - href: 'file://./../src/neuvector/oscal-component.yaml' - href: 'file://./../src/prometheus-stack/oscal-component.yaml' - - href: 'file://./../src/promtail/oscal-component.yaml' + - href: 'file://./../src/vector/oscal-component.yaml' - href: 'file://./../src/velero/oscal-component.yaml' capabilities: diff --git a/docs/application-baseline.md b/docs/application-baseline.md index ca32e0fe0..d2410c243 100644 --- a/docs/application-baseline.md +++ b/docs/application-baseline.md @@ -18,7 +18,7 @@ For optimal deployment and operational efficiency, it is important to deliver a | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | **Service Mesh** | **[Istio](https://istio.io/):** A powerful service mesh that provides traffic management, load balancing, security, and observability features. | | **Monitoring** | **[Metrics Server](https://kubernetes-sigs.github.io/metrics-server/):** Provides container resource utilization metrics API for Kubernetes clusters. Metrics server is an optional (non-default) component since most Kubernetes distros provide it by default.

**[Prometheus](https://prometheus.io/):** Scrapes Metrics Server API and application metrics and stores the data in a time-series database for insights into application health and performance.

**[Grafana](https://grafana.com/grafana/):** Provides visualization and alerting capabilities based on Prometheus's time-series database of metrics. | -| **Logging** | **[Promtail](https://grafana.com/docs/loki/latest/send-data/promtail/):** A companion agent that efficiently gathers and sends container logs to Loki, simplifying log monitoring, troubleshooting, and compliance auditing, enhancing the overall observability of the mission environment.

**[Loki](https://grafana.com/docs/loki/latest/):** A log aggregation system that allows users to store, search, and analyze logs across their applications. | +| **Logging** | **[Vector](https://vector.dev/):** A companion agent that efficiently gathers and sends container logs to Loki, simplifying log monitoring, troubleshooting, and compliance auditing, enhancing the overall observability of the mission environment.

**[Loki](https://grafana.com/docs/loki/latest/):** A log aggregation system that allows users to store, search, and analyze logs across their applications. | | **Security and Compliance** | **[NeuVector](https://open-docs.neuvector.com/):** Offers container-native security, protecting applications against threats and vulnerabilities.

**[Pepr](https://pepr.dev/):** UDS policy engine and operator for enhanced security and compliance.| | **Identity and Access Management** | **[Keycloak](https://www.keycloak.org/):** A robust open-source Identity and Access Management solution, providing centralized authentication, authorization, and user management for enhanced security and control over access to mission-critical resources.| | **Backup and Restore** | **[Velero](https://velero.io/):** Provides backup and restore capabilities for Kubernetes clusters, ensuring data protection and disaster recovery.| diff --git a/docs/configuration/resource-configuration-and-ha.md b/docs/configuration/resource-configuration-and-ha.md index 0f8f30213..df0daa9fe 100644 --- a/docs/configuration/resource-configuration-and-ha.md +++ b/docs/configuration/resource-configuration-and-ha.md @@ -40,9 +40,19 @@ To scale Grafana for high availability, its database must be externalized (see [ ## Logging -### Promtail +### Vector -By default Promtail runs as a daemonset, automatically scaling across all nodes to ensure logs are captured from each host. Typically Promtail does not need any other modifications, but you can customize its resource configuration by overriding the `resources` helm value (using the component and chart name of `promtail`). +By default Vector runs as a daemonset, automatically scaling across all nodes to ensure logs are captured from each host. Typically Vector does not need any other modifications, but you can customize its resource configuration by overriding the `resources` helm value (using the component and chart name of `vector`). Vector recommends the below resourcing when running in production: + +```yaml +resources: + requests: + memory: "64Mi" + cpu: "500m" + limits: + memory: "1024Mi" + cpu: "6000m" +``` ### Loki diff --git a/packages/standard/zarf.yaml b/packages/standard/zarf.yaml index 9f112e7dc..decbd34dd 100644 --- a/packages/standard/zarf.yaml +++ b/packages/standard/zarf.yaml @@ -76,11 +76,11 @@ components: import: path: ../../src/prometheus-stack - # Promtail - - name: promtail + # Vector + - name: vector required: true import: - path: ../../src/promtail + path: ../../src/vector # Grafana - name: grafana diff --git a/renovate.json b/renovate.json index 999119d32..a4b2a0a68 100644 --- a/renovate.json +++ b/renovate.json @@ -57,9 +57,9 @@ "commitMessageTopic": "istio" }, { - "matchFileNames": ["src/promtail/**"], - "groupName": "promtail", - "commitMessageTopic": "promtail" + "matchFileNames": ["src/vector/**"], + "groupName": "vector", + "commitMessageTopic": "vector" }, { "matchFileNames": ["src/velero/**"], diff --git a/src/loki/chart/templates/uds-package.yaml b/src/loki/chart/templates/uds-package.yaml index 8f30a3d0c..1cb2e25fc 100644 --- a/src/loki/chart/templates/uds-package.yaml +++ b/src/loki/chart/templates/uds-package.yaml @@ -37,12 +37,12 @@ spec: - direction: Ingress selector: app.kubernetes.io/name: loki - remoteNamespace: promtail + remoteNamespace: vector remoteSelector: - app.kubernetes.io/name: promtail + app.kubernetes.io/name: vector ports: - 8080 - description: "Promtail Log Storage" + description: "Vector Log Storage" # Todo: wide open for now for pushing to s3 - direction: Egress diff --git a/src/pepr/operator/controllers/exemptions/exemptions.spec.ts b/src/pepr/operator/controllers/exemptions/exemptions.spec.ts index 8c276d879..006124691 100644 --- a/src/pepr/operator/controllers/exemptions/exemptions.spec.ts +++ b/src/pepr/operator/controllers/exemptions/exemptions.spec.ts @@ -19,13 +19,13 @@ const prometheusMatcher = { name: "^neuvector-prometheus-exporter-pod.*", kind: MatcherKind.Pod, }; -const promtailMatcher = { namespace: "promtail", name: "^promtail-.*", kind: MatcherKind.Pod }; +const vectorMatcher = { namespace: "vector", name: "^vector-.*", kind: MatcherKind.Pod }; const exemption1UID = "exemption-1-uid"; const exemption2UID = "exemption-2-uid"; const storedEnforcerMatcher = { ...enforcerMatcher, owner: exemption1UID }; const storedControllerMatcher = { ...controllerMatcher, owner: exemption1UID }; const storedPrometheusMatcher = { ...prometheusMatcher, owner: exemption1UID }; -const storedPromtailMatcher = { ...promtailMatcher, owner: exemption2UID }; +const storedVectorMatcher = { ...vectorMatcher, owner: exemption2UID }; const neuvectorMockExemption = { metadata: { uid: exemption1UID, @@ -89,7 +89,7 @@ describe("Test processExemptions() no duplicate matchers in same CR", () => { // remove RequireNonRootUser from enforcerMatcher // remove prometheusMatcher // add DisallowHostNamespaces to controllerMatcher - // add promtailMatcher with RequireNonRootUser + // add vectorMatcher with RequireNonRootUser const updatedNeuvectorExemption = { metadata: { uid: exemption1UID, @@ -109,7 +109,7 @@ describe("Test processExemptions() no duplicate matchers in same CR", () => { ], }, { - matcher: promtailMatcher, + matcher: vectorMatcher, policies: [Policy.RequireNonRootUser], }, ], @@ -119,7 +119,7 @@ describe("Test processExemptions() no duplicate matchers in same CR", () => { processExemptions(neuvectorMockExemption, WatchPhase.Added); processExemptions(updatedNeuvectorExemption, WatchPhase.Modified); expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([ - { ...storedPromtailMatcher, owner: exemption1UID }, + { ...storedVectorMatcher, owner: exemption1UID }, ]); expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([ storedEnforcerMatcher, @@ -359,14 +359,14 @@ describe("Test processExemptions(); phase DELETED", () => { }); it("Does not remove exemptions set by separate CR from the one being deleted", async () => { - const promtailMockExemption = { + const vectorMockExemption = { metadata: { uid: exemption2UID, }, spec: { exemptions: [ { - matcher: promtailMatcher, + matcher: vectorMatcher, policies: [ Policy.DisallowPrivileged, Policy.DropAllCapabilities, @@ -378,12 +378,12 @@ describe("Test processExemptions(); phase DELETED", () => { } as Exemption; processExemptions(neuvectorMockExemption, WatchPhase.Added); - processExemptions(promtailMockExemption, WatchPhase.Added); + processExemptions(vectorMockExemption, WatchPhase.Added); processExemptions(neuvectorMockExemption, WatchPhase.Deleted); - expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([storedPromtailMatcher]); - expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([storedPromtailMatcher]); - expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([storedPromtailMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([storedVectorMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([storedVectorMatcher]); + expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([storedVectorMatcher]); }); it("Does not delete duplicate exemptions if set by separate CRs", async () => { @@ -447,28 +447,28 @@ describe("Test processExemptions(); phase DELETED", () => { }, } as Exemption; - const promtailMockExemption = { + const vectorMockExemption = { metadata: { uid: exemption2UID, }, spec: { exemptions: [ { - matcher: promtailMatcher, + matcher: vectorMatcher, policies: [Policy.DisallowPrivileged], }, ], }, } as Exemption; - const promtailUpdatedMockExemption = { + const vectorUpdatedMockExemption = { metadata: { uid: exemption2UID, }, spec: { exemptions: [ { - matcher: promtailMatcher, + matcher: vectorMatcher, policies: [Policy.DisallowPrivileged, Policy.RequireNonRootUser], }, ], @@ -476,14 +476,14 @@ describe("Test processExemptions(); phase DELETED", () => { } as Exemption; processExemptions(neuvectorMockExemption, WatchPhase.Added); - processExemptions(promtailMockExemption, WatchPhase.Added); - processExemptions(promtailUpdatedMockExemption, WatchPhase.Modified); + processExemptions(vectorMockExemption, WatchPhase.Added); + processExemptions(vectorUpdatedMockExemption, WatchPhase.Modified); expect(ExemptionStore.getByPolicy(Policy.RequireNonRootUser)).toEqual([ storedEnforcerMatcher, - storedPromtailMatcher, + storedVectorMatcher, ]); expect(ExemptionStore.getByPolicy(Policy.DropAllCapabilities)).toEqual([storedEnforcerMatcher]); - expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([storedPromtailMatcher]); + expect(ExemptionStore.getByPolicy(Policy.DisallowPrivileged)).toEqual([storedVectorMatcher]); }); }); diff --git a/src/pepr/policies/exemptions/index.spec.ts b/src/pepr/policies/exemptions/index.spec.ts index 2ab36dd25..3f8faa429 100644 --- a/src/pepr/policies/exemptions/index.spec.ts +++ b/src/pepr/policies/exemptions/index.spec.ts @@ -34,7 +34,7 @@ describe("test registering exemptions", () => { const req = { Raw: { metadata: { - name: "promtail", + name: "vector", namespace: "monitoring", }, }, diff --git a/src/promtail/README.md b/src/promtail/README.md deleted file mode 100644 index 447959057..000000000 --- a/src/promtail/README.md +++ /dev/null @@ -1 +0,0 @@ -## Promtail diff --git a/src/promtail/chart/.helmignore b/src/promtail/chart/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/src/promtail/chart/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/src/promtail/chart/Chart.yaml b/src/promtail/chart/Chart.yaml deleted file mode 100644 index 84403fdd5..000000000 --- a/src/promtail/chart/Chart.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v2 -name: uds-promtail-config -description: Promtail configuration for UDS - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 diff --git a/src/promtail/chart/templates/_helpers.tpl b/src/promtail/chart/templates/_helpers.tpl deleted file mode 100644 index e2736937a..000000000 --- a/src/promtail/chart/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "uds-promtail-config.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "uds-promtail-config.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "uds-promtail-config.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "uds-promtail-config.labels" -}} -helm.sh/chart: {{ include "uds-promtail-config.chart" . }} -{{ include "uds-promtail-config.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "uds-promtail-config.selectorLabels" -}} -app.kubernetes.io/name: {{ include "uds-promtail-config.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "uds-promtail-config.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "uds-promtail-config.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/src/promtail/chart/templates/service.yaml b/src/promtail/chart/templates/service.yaml deleted file mode 100644 index 23c6a4429..000000000 --- a/src/promtail/chart/templates/service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Upstream chart can create this service but it is conditionally tied to the serviceMonitor which would cause errors in single package testing -# This would be resolved by https://github.com/grafana/helm-charts/pull/3083 when merged and released -apiVersion: v1 -kind: Service -metadata: - name: promtail-metrics - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: promtail -spec: - clusterIP: None - ports: - - name: http-metrics - port: 3101 - targetPort: http-metrics - protocol: TCP - selector: - app.kubernetes.io/name: promtail diff --git a/src/promtail/chart/templates/uds-exemption.yaml b/src/promtail/chart/templates/uds-exemption.yaml deleted file mode 100644 index 9b8bca9cf..000000000 --- a/src/promtail/chart/templates/uds-exemption.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: uds.dev/v1alpha1 -kind: Exemption -metadata: - name: promtail - namespace: uds-policy-exemptions -spec: - exemptions: - - policies: - - DisallowPrivileged - - RequireNonRootUser - - RestrictSELinuxType - - RestrictHostPathWrite - - RestrictVolumeTypes - matcher: - namespace: promtail - name: "^promtail-.*" - title: "promtail exemptions" - description: "Promtail mounts the following hostPaths: - - `/var/log/pods`: to tail pod logs - - `/var/lib/docker/containers`: to tail container logs - - `/run/promtail`: for Promtail's buffering and persistent state - Since logs can have sensitive information, it is better to exclude - Promtail from the policy than add the paths as allowable mounts - https://github.com/grafana/helm-charts/blob/main/charts/promtail/templates/daemonset.yaml#L120" diff --git a/src/promtail/chart/templates/uds-package.yaml b/src/promtail/chart/templates/uds-package.yaml deleted file mode 100644 index 1a66b8490..000000000 --- a/src/promtail/chart/templates/uds-package.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: uds.dev/v1alpha1 -kind: Package -metadata: - name: promtail - namespace: {{ .Release.Namespace }} -spec: - monitor: - - selector: - app.kubernetes.io/name: promtail - targetPort: 3101 - portName: http-metrics - description: Metrics - - network: - allow: - - direction: Ingress - selector: - app.kubernetes.io/name: promtail - remoteNamespace: monitoring - remoteSelector: - app.kubernetes.io/name: prometheus - port: 3101 - description: "Prometheus Metrics" - - - direction: Egress - selector: - app.kubernetes.io/name: promtail - remoteGenerated: KubeAPI - - - direction: Egress - remoteNamespace: tempo - remoteSelector: - app.kubernetes.io/name: tempo - port: 9411 - description: "Tempo" - - - direction: Egress - selector: - app.kubernetes.io/name: promtail - remoteNamespace: loki - remoteSelector: - app.kubernetes.io/name: loki - port: 8080 - description: "Write Logs to Loki" diff --git a/src/promtail/chart/values.yaml b/src/promtail/chart/values.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/promtail/common/zarf.yaml b/src/promtail/common/zarf.yaml deleted file mode 100644 index bb7efd23a..000000000 --- a/src/promtail/common/zarf.yaml +++ /dev/null @@ -1,32 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: uds-core-promtail-common - description: "UDS Core Promtail Common" - url: "https://grafana.com/docs/loki/latest/" - -components: - - name: promtail - required: true - charts: - - name: uds-promtail-config - namespace: promtail - version: 0.1.0 - localPath: ../chart - - name: promtail - url: https://grafana.github.io/helm-charts/ - version: 6.16.5 - namespace: promtail - gitPath: charts/promtail - valuesFiles: - - ../values/values.yaml - actions: - onDeploy: - after: - - description: Validate Promtail Package - maxTotalSeconds: 300 - wait: - cluster: - kind: Packages - name: promtail - namespace: promtail - condition: "'{.status.phase}'=Ready" diff --git a/src/promtail/oscal-component.yaml b/src/promtail/oscal-component.yaml deleted file mode 100644 index 94635da4e..000000000 --- a/src/promtail/oscal-component.yaml +++ /dev/null @@ -1,268 +0,0 @@ -component-definition: - uuid: ff959bdb-7be9-49b3-9dc2-c41b34e7017d - metadata: - title: Promtail - last-modified: "2024-01-31T16:44:35Z" - version: "20240132" - oscal-version: 1.1.2 - parties: - - uuid: f3cf70f8-ba44-4e55-9ea3-389ef24847d3 - type: organization - name: Defense Unicorns - links: - - href: https://defenseunicorns.com - rel: website - components: - - uuid: 3ca1e9a3-a566-48d1-93af-200abd1245e3 - type: software - title: Promtail - description: | - Log collector - purpose: Collects logs from the cluster - responsible-roles: - - role-id: provider - party-uuids: - - f3cf70f8-ba44-4e55-9ea3-389ef24847d3 - control-implementations: - - uuid: d2afb4c4-2cd8-5305-a6cc-d1bc7b388d0c - source: https://raw.githubusercontent.com/GSA/fedramp-automation/93ca0e20ff5e54fc04140613476fba80f08e3c7d/dist/content/rev5/baselines/json/FedRAMP_rev5_HIGH-baseline-resolved-profile_catalog.json - description: Controls implemented by Promtail for inheritance by applications - implemented-requirements: - - uuid: 954ba9c8-452c-4503-a43f-c880a01b828d - control-id: ac-6.9 - description: >- - # Control Description - Misuse of privileged functions, either intentionally or unintentionally by authorized users, or by unauthorized external entities that have compromised information system accounts, is a serious and ongoing concern and can have significant adverse impacts on organizations. - Auditing the use of privileged functions is one way to detect such misuse, and in doing so, help mitigate the risk from insider threats and the advanced persistent threat (APT). - - # Control Implementation - Promtail can be configured to collect all logs from Kubernetes and underlying operating systems, allowing the aggregation of privileged function calls. - remarks: This control is fully implemented by this tool. - links: - - href: "#98b97ec9-a9ce-4444-83d8-71066270a424" - rel: reference - text: Lula Validation - - href: "#fbe5855d-b4ea-4ff5-9f0d-5901d620577a" - rel: reference - text: Lula Validation - - - uuid: 2a25a5a4-4fbc-4fbc-88e3-2e34ddc3fb0e - control-id: au-2 - description: >- - # Control Description - An event is any observable occurrence in an organizational information system. - Organizations identify audit events as those events which are significant and relevant to the security of information systems and the environments in which those systems operate in order to meet specific and ongoing audit needs. - Audit events can include, for example, password changes, failed logons, or failed accesses related to information systems, administrative privilege usage, PIV credential usage, or third-party credential usage. - In determining the set of auditable events, organizations consider the auditing appropriate for each of the security controls to be implemented. - To balance auditing requirements with other information system needs, this control also requires identifying that subset of auditable events that are audited at a given point in time. - - # Control Implementation - Logging daemons are present on each node that BigBang is installed on. Out of the box, the following events are captured: - * all containers emitting to STDOUT or STDERR (captured by container runtime translating container logs to /var/log/containers). - * all kubernetes api server requests. - * all events emitted by the kubelet. - remarks: This control is fully implemented by this tool. - links: - - href: "#98b97ec9-a9ce-4444-83d8-71066270a424" - rel: reference - text: Lula Validation - - href: "#0be7345d-e9d3-4248-9c14-5fed8e7bfa01" - rel: reference - text: Lula Validation - - - uuid: 762604db-77ec-415f-8728-c296873ab48b - control-id: au-3 - description: >- - # Control Description - Audit record content that may be necessary to satisfy the requirement of this control, includes, for example, time stamps, source and destination addresses, user/process identifiers, event descriptions, success/fail indications, filenames involved, and access control or flow control rules invoked. - Event outcomes can include indicators of event success or failure and event-specific results (e.g., the security state of the information system after the event occurred). - - # Control Implementation - Logs are captured by promtail from the node. The node logs will contain the necessary log data from all pods/applications inside the selected nodes. - Validating `logfmt` as the config.logFormat would be the goal. This is currently a secret mounted to /etc/promtail/promtail.yaml in the promtail container. We will ensure the promtail.yaml file is at a minimum the target config. - https://grafana.com/docs/loki/latest/send-data/promtail/stages/logfmt/ - remarks: This control is fully implemented by this tool. - links: - - href: "#98b97ec9-a9ce-4444-83d8-71066270a424" - rel: reference - text: Lula Validation - - href: "#9bfc68e0-381a-4006-9f68-c293e3b20cee" - rel: reference - text: Lula Validation - - - uuid: 9ad7ddfb-4701-4c34-88f7-9d85abb13d60 - control-id: au-8 - description: >- - # Control Description - Time stamps generated by the information system include date and time. - Time is commonly expressed in Coordinated Universal Time (UTC), a modern continuation of Greenwich Mean Time (GMT), or local time with an offset from UTC. - Granularity of time measurements refers to the degree of synchronization between information system clocks and reference clocks, for example, clocks synchronizing within hundreds of milliseconds or within tens of milliseconds. - Organizations may define different time granularities for different system components. - Time service can also be critical to other security capabilities such as access control and identification and authentication, depending on the nature of the mechanisms used to support those capabilities. - - # Control Implementation - Records captured by the logging daemon are enriched to ensure the following are always present: - * time of the event (UTC). - * source of event (pod, namespace, container id). - Applications are responsible for providing all other information. - Validating `logfmt` as the config.logFormat would be the goal. This is currently a secret mounted to /etc/promtail/promtail.yaml in the promtail container. We will ensure the promtail.yaml file is at a minimum the target config. - https://grafana.com/docs/loki/latest/send-data/promtail/stages/logfmt/ - remarks: This control is fully implemented by this tool. - links: - - href: "#98b97ec9-a9ce-4444-83d8-71066270a424" - rel: reference - text: Lula Validation - - href: "#9bfc68e0-381a-4006-9f68-c293e3b20cee" - rel: reference - text: Lula Validation - props: - - name: framework - ns: https://docs.lula.dev/oscal/ns - value: il4 - back-matter: - resources: - - uuid: D552C935-E40C-4A03-B5CC-4605EBD95B6D - title: Promtail - rlinks: - - href: https://grafana.com/docs/loki/latest/clients/promtail/ - - uuid: 211C474B-E11A-4DD2-8075-50CDAC507CDC - title: Big Bang Promtail package - rlinks: - - href: https://repo1.dso.mil/platform-one/big-bang/apps/sandbox/promtail - - uuid: 98b97ec9-a9ce-4444-83d8-71066270a424 - title: Lula Validation - rlinks: - - href: lula.dev - remarks: Validation health check - description: >- - target: - provider: opa - domain: kubernetes - payload: - resources: - - name: daemonsets - resourceRule: - Group: apps - Version: v1 - Resource: daemonsets - Namespaces: [promtail] - rego: | - package validate - - import future.keywords.every - - validate { - every daemonset in input.daemonsets { - daemonset.kind == "DaemonSet" - podsScheduled := daemonset.status.desiredNumberScheduled - numberAvailable := daemonset.status.numberAvailable - numberReady := daemonset.status.numberReady - podsScheduled == numberAvailable - numberAvailable == numberReady - } - } - - uuid: fbe5855d-b4ea-4ff5-9f0d-5901d620577a - title: Lula Validation - remarks: Log the execution of privileged functions. - rlinks: - - href: lula.dev - description: >- - target: - provider: opa - domain: kubernetes - payload: - resources: - - name: pods - resourceRule: - Group: - Version: v1 - Resource: pods - Namespaces: [promtail] - rego: | - package validate - - import future.keywords.every - - validate { - every pod in input.pods { - volumes := pod.spec.volumes - - some volume in volumes - volume.name == "varlog" - volume.hostPath.path == "/var/log" - } - } - - uuid: 0be7345d-e9d3-4248-9c14-5fed8e7bfa01 - title: Lula Validation - remarks: - a. Identify the types of events that the system is capable of logging in support of the audit function for organization-defined event types that the system is capable of logging; - b. Coordinate the event logging function with other organizational entities requiring audit-related information to guide and inform the selection criteria for events to be logged; - c. Specify the following event types for logging within the system organization-defined event types (subset of the event types defined in AU-2a.) along with the frequency of (or situation requiring) logging for each identified event type; - d. Provide a rationale for why the event types selected for logging are deemed to be adequate to support after-the-fact investigations of incidents; and - e. Review and update the event types selected for logging on an organization-defined frequency. - rlinks: - - href: lula.dev - description: >- - target: - provider: opa - domain: kubernetes - payload: - resources: - - name: pods - resourceRule: - Group: - Version: v1 - Resource: pods - Namespaces: [promtail] - rego: | - package validate - - import future.keywords.every - - validate { - every pod in input.pods { - volumes := pod.spec.volumes - - some volume in volumes - volume.name == "pods" - volume.hostPath.path == "/var/log/pods" - } - } - - uuid: 9bfc68e0-381a-4006-9f68-c293e3b20cee - title: Lula Validation - remarks: Ensure that audit records contain information that establishes the following; - a. What type of event occurred; - b. When the event occurred; - c. Where the event occurred; - d. Source of the event; - e. Outcome of the event; and - f. Identity of any individuals, subjects, or objects/entities associated with the event. - rlinks: - - href: lula.dev - description: >- - target: - provider: opa - domain: kubernetes - payload: - resources: - - name: pods - resourceRule: - Group: - Version: v1 - Resource: pods - Namespaces: [promtail] - rego: | - package validate - - import future.keywords.every - - validate { - every pod in input.pods { - containers := pod.spec.containers - - some container in containers - container.name == "promtail" - some i - container.args[i] == "-config.file=/etc/promtail/promtail.yaml" - } - } diff --git a/src/promtail/tasks.yaml b/src/promtail/tasks.yaml deleted file mode 100644 index 8117f590a..000000000 --- a/src/promtail/tasks.yaml +++ /dev/null @@ -1,10 +0,0 @@ -tasks: - - name: validate - actions: - - description: Validate promtail - wait: - cluster: - kind: Pod - name: app.kubernetes.io/instance=promtail - namespace: promtail - condition: Ready diff --git a/src/promtail/values/registry1-values.yaml b/src/promtail/values/registry1-values.yaml deleted file mode 100644 index 6dec37593..000000000 --- a/src/promtail/values/registry1-values.yaml +++ /dev/null @@ -1,10 +0,0 @@ -image: - registry: registry1.dso.mil - repository: ironbank/opensource/grafana/promtail - tag: v3.1.1 -sidecar: - configReloader: - image: - registry: registry1.dso.mil - repository: ironbank/opensource/jimmidyson/configmap-reload - tag: v0.13.1 diff --git a/src/promtail/values/unicorn-values.yaml b/src/promtail/values/unicorn-values.yaml deleted file mode 100644 index c2248c2a6..000000000 --- a/src/promtail/values/unicorn-values.yaml +++ /dev/null @@ -1,10 +0,0 @@ -image: - registry: cgr.dev - repository: du-uds-defenseunicorns/promtail - tag: 3.1.1 -sidecar: - configReloader: - image: - registry: cgr.dev - repository: du-uds-defenseunicorns/configmap-reload-fips - tag: 0.13.1 diff --git a/src/promtail/values/upstream-values.yaml b/src/promtail/values/upstream-values.yaml deleted file mode 100644 index 9c9dc6f40..000000000 --- a/src/promtail/values/upstream-values.yaml +++ /dev/null @@ -1,10 +0,0 @@ -image: - registry: docker.io - repository: grafana/promtail - tag: 3.1.1 -sidecar: - configReloader: - image: - registry: ghcr.io - repository: jimmidyson/configmap-reload - tag: v0.13.1 diff --git a/src/promtail/values/values.yaml b/src/promtail/values/values.yaml deleted file mode 100644 index d7bb9af71..000000000 --- a/src/promtail/values/values.yaml +++ /dev/null @@ -1,116 +0,0 @@ -config: - clients: - - url: 'http://loki-gateway.loki.svc.cluster.local:80/loki/api/v1/push' - - snippets: - scrapeConfigs: | - # Upstream Defaults https://github.com/grafana/helm-charts/blob/main/charts/promtail/values.yaml - # See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference - - job_name: kubernetes-pods - pipeline_stages: - {{- toYaml .Values.config.snippets.pipelineStages | nindent 4 }} - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: - - __meta_kubernetes_pod_controller_name - regex: ([0-9a-z-.]+?)(-[0-9a-f]{8,10})? - action: replace - target_label: __tmp_controller_name - - source_labels: - - __meta_kubernetes_pod_label_app_kubernetes_io_name - - __meta_kubernetes_pod_label_app - - __tmp_controller_name - - __meta_kubernetes_pod_name - regex: ^;*([^;]+)(;.*)?$ - action: replace - target_label: app - - source_labels: - - __meta_kubernetes_pod_label_app_kubernetes_io_instance - - __meta_kubernetes_pod_label_instance - regex: ^;*([^;]+)(;.*)?$ - action: replace - target_label: instance - - source_labels: - - __meta_kubernetes_pod_label_app_kubernetes_io_component - - __meta_kubernetes_pod_label_component - regex: ^;*([^;]+)(;.*)?$ - action: replace - target_label: component - {{- if .Values.config.snippets.addScrapeJobLabel }} - - replacement: kubernetes-pods - target_label: scrape_job - {{- end }} - {{- toYaml .Values.config.snippets.common | nindent 4 }} - {{- with .Values.config.snippets.extraRelabelConfigs }} - {{- toYaml . | nindent 4 }} - {{- end }} - # UDS CORE Defaults - - job_name: systemd-messages - static_configs: - - targets: [localhost] - labels: - job: varlogs - host: "${NODE_HOSTNAME}" - __path__: /var/log/* - relabel_configs: - - source_labels: - - __journal_systemd_unit - target_label: systemd_unit - - source_labels: - - __journal_hostname - target_label: nodename - - source_labels: - - __journal_syslog_identifier - target_label: syslog_identifier - - job_name: kubernetes-logs - static_configs: - - targets: [localhost] - labels: - job: kubernetes-logs - host: "${NODE_HOSTNAME}" - __path__: /var/log/kubernetes/**/*.log - -containerSecurityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - privileged: false - readOnlyRootFilesystem: true - runAsUser: 0 - seLinuxOptions: - type: spc_t -extraArgs: - - '-config.expand-env=true' - -extraEnv: - - name: NODE_HOSTNAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - -extraVolumes: - - hostPath: - path: /var/log - name: varlog - - hostPath: - path: /etc - name: machine-id - -extraVolumeMounts: - - mountPath: /var/log - name: varlog - readOnly: true - - mountPath: /etc/machine-id - name: machine-id - readOnly: true - subPath: machine-id - -resources: - limits: - cpu: 500m - memory: 750Mi - requests: - cpu: 100m - memory: 256Mi diff --git a/src/promtail/zarf.yaml b/src/promtail/zarf.yaml deleted file mode 100644 index 69354c754..000000000 --- a/src/promtail/zarf.yaml +++ /dev/null @@ -1,51 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: uds-core-promtail - description: "UDS Core Promtail" - url: "https://grafana.com/docs/loki/latest/" - -components: - - name: promtail - required: true - description: "Deploy Promtail" - only: - flavor: upstream - import: - path: common - charts: - - name: promtail - valuesFiles: - - values/upstream-values.yaml - images: - - ghcr.io/jimmidyson/configmap-reload:v0.13.1 - - docker.io/grafana/promtail:3.1.1 - - - name: promtail - required: true - description: "Deploy Promtail" - only: - flavor: registry1 - import: - path: common - charts: - - name: promtail - valuesFiles: - - values/registry1-values.yaml - images: - - registry1.dso.mil/ironbank/opensource/jimmidyson/configmap-reload:v0.13.1 - - registry1.dso.mil/ironbank/opensource/grafana/promtail:v3.1.1 - - - name: promtail - required: true - description: "Deploy Promtail" - only: - flavor: unicorn - import: - path: common - charts: - - name: promtail - valuesFiles: - - values/unicorn-values.yaml - images: - - cgr.dev/du-uds-defenseunicorns/configmap-reload-fips:0.13.1 - - cgr.dev/du-uds-defenseunicorns/promtail:3.1.1 diff --git a/src/vector/chart/templates/uds-exemption.yaml b/src/vector/chart/templates/uds-exemption.yaml index e70a7572a..0c6032102 100644 --- a/src/vector/chart/templates/uds-exemption.yaml +++ b/src/vector/chart/templates/uds-exemption.yaml @@ -6,7 +6,6 @@ metadata: spec: exemptions: - policies: - - DisallowPrivileged - RequireNonRootUser - RestrictSELinuxType - RestrictHostPathWrite @@ -16,8 +15,7 @@ spec: name: "^vector-.*" title: "vector exemptions" description: "Vector mounts the following hostPaths: - - `/var/log/pods`: to tail pod logs - - `/var/lib/docker/containers`: to tail container logs - - `/run/vector`: for Vector's buffering and persistent state + - `/var/log`: to tail logs + - `/var/lib/vector`: for Vector's buffering and persistent state Since logs can have sensitive information, it is better to exclude - Vector from the policy than add the paths as allowable mounts + Vector from the policy than add the paths as allowable mounts" diff --git a/src/vector/chart/templates/uds-package.yaml b/src/vector/chart/templates/uds-package.yaml index ff39b0ce0..6cece0d01 100644 --- a/src/vector/chart/templates/uds-package.yaml +++ b/src/vector/chart/templates/uds-package.yaml @@ -4,36 +4,13 @@ metadata: name: vector namespace: {{ .Release.Namespace }} spec: - monitor: - - selector: - app.kubernetes.io/name: vector - targetPort: 3101 - portName: http-metrics - description: Metrics - network: allow: - - direction: Ingress - selector: - app.kubernetes.io/name: vector - remoteNamespace: monitoring - remoteSelector: - app.kubernetes.io/name: prometheus - port: 3101 - description: "Prometheus Metrics" - - direction: Egress selector: app.kubernetes.io/name: vector remoteGenerated: KubeAPI - - direction: Egress - remoteNamespace: tempo - remoteSelector: - app.kubernetes.io/name: tempo - port: 9411 - description: "Tempo" - - direction: Egress selector: app.kubernetes.io/name: vector diff --git a/src/vector/common/zarf.yaml b/src/vector/common/zarf.yaml index 118d920b3..65734e2b4 100644 --- a/src/vector/common/zarf.yaml +++ b/src/vector/common/zarf.yaml @@ -21,6 +21,12 @@ components: - ../values/values.yaml actions: onDeploy: + before: + - description: Remove Promtail Components if necessary + mute: true + cmd: | + ./zarf package remove core --components promtail --confirm || true # Ensure this doesn't error on installs and upgrades when Promtail no longer exists + ./zarf tools kubectl delete ns promtail || true # Ensure this doesn't error on installs and upgrades when Promtail no longer exists after: - description: Validate Vector Package maxTotalSeconds: 300 diff --git a/src/vector/values/values.yaml b/src/vector/values/values.yaml index c0131e0c5..769a34687 100644 --- a/src/vector/values/values.yaml +++ b/src/vector/values/values.yaml @@ -1,24 +1,84 @@ # Run as an agent daemonset role: "Agent" -# todo: all this customConfig: - data_dir: /vector-data-dir - api: - enabled: true - address: 127.0.0.1:8686 - playground: false + data_dir: /var/lib/vector sources: - vector: - address: 0.0.0.0:6000 - type: vector - version: "2" + pod_logs: + type: "kubernetes_logs" + oldest_first: true + node_logs: + type: "file" + include: ["/var/log/*"] + oldest_first: true + k8s_logs: + type: "file" + include: ["/var/log/kubernetes/**/*.log"] + oldest_first: true + internal_metrics: + type: internal_metrics + + transforms: + pod_logs_labelled: + type: remap + inputs: ["pod_logs"] + source: | + if !exists(.kubernetes.pod_labels.app) { + if exists(.kubernetes.pod_labels."app.kubernetes.io/name") { + .kubernetes.pod_labels.app = .kubernetes.pod_labels."app.kubernetes.io/name" + } else if exists(.kubernetes.pod_owner) { + .kubernetes.pod_labels.app = .kubernetes.pod_owner + } else { + .kubernetes.pod_labels.app = .kubernetes.pod_name + } + } + host_logs_labelled: + type: remap + inputs: ["node_logs", "k8s_logs"] + source: | + .node_name = "${NODE_HOSTNAME}" + if contains(string!(.file), "/var/log/kubernetes/") { + .job = "kubernetes-logs" + } else { + .job = "varlogs" + } + sinks: - stdout: - type: console - inputs: [vector] + loki_pod: + type: "loki" + inputs: ["pod_logs_labelled"] + endpoint: "http://loki-gateway.loki.svc.cluster.local:80" + path: "/loki/api/v1/push" encoding: - codec: json + codec: "raw_message" + labels: + namespace: '{{`{{ kubernetes.pod_namespace }}`}}' + app: '{{`{{ kubernetes.pod_labels.app }}`}}' + job: '{{`{{ kubernetes.pod_namespace }}`}}/{{`{{ kubernetes.pod_labels.app }}`}}' + container: '{{`{{ kubernetes.container_name }}`}}' + host: '{{`{{ kubernetes.pod_node_name }}`}}' + file: '{{`{{ file }}`}}' + buffer: + type: disk + max_size: 1073741824 # 1GiB + loki_host: + type: "loki" + inputs: ["host_logs_labelled"] + endpoint: "http://loki-gateway.loki.svc.cluster.local:80" + path: "/loki/api/v1/push" + encoding: + codec: "raw_message" + labels: + job: '{{`{{ job }}`}}' + host: '{{`{{ node_name }}`}}' + file: '{{`{{ file }}`}}' + buffer: + type: disk + max_size: 1073741824 # 1GiB + prom_exporter: + type: prometheus_exporter + inputs: [internal_metrics] + address: 0.0.0.0:9090 persistence: enabled: true @@ -28,3 +88,20 @@ persistence: podMonitor: enabled: true +service: + ports: + - name: prom-exporter + port: 9090 + protocol: TCP + +securityContext: + readOnlyRootFilesystem: true + runAsUser: 0 + seLinuxOptions: + type: spc_t + +env: + - name: NODE_HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName From 64014c11a054112e4a187cb40febf3f76136b013 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 4 Sep 2024 11:49:53 -0600 Subject: [PATCH 03/17] chore: docs and netpols --- src/vector/README.md | 37 ++++++- src/vector/chart/templates/uds-package.yaml | 9 ++ src/vector/pepr.log | 102 -------------------- 3 files changed, 45 insertions(+), 103 deletions(-) delete mode 100644 src/vector/pepr.log diff --git a/src/vector/README.md b/src/vector/README.md index 80f646b44..f40cacb2e 100644 --- a/src/vector/README.md +++ b/src/vector/README.md @@ -1 +1,36 @@ -## Vector +# Vector + +Vector is a lightweight tool for building observability pipelines, built and maintained primarily by Datadog. Within UDS Core it is primarily used for log collection and shipping to destinations (like Loki and S3). + +## Switching from Promtail to Vector + +Within UDS Core we have made the decision to switch from Promtail (historically the log collector/shipper of choice) to Vector. The below contains primary motivating factors and impacts of this choice. + +### Motivations + +Promtail has historically been the tool of choice for log collection/shipping when using Loki. It provides a very lightweight layer to scrape logs from pods and hosts, label them with additional metadata, and ship them to Loki. + +One of the main issues that has arisen with Promtail is its limited output/export options. Promtail only supports sending logs to one or more Loki instances. A common requirement in production environments is to ship logs to a secondary destination for collection/analysis by security teams and SIEM tools. Promtail is currently listed as [feature complete](https://grafana.com/docs/loki/latest/send-data/promtail/) so there is no expectation that additional export functionality would be added. + +### Goals and Options + +In choosing an alternative to Promtail we have a few primary objectives: +1. Chosen tool must be capable of gathering host and pod logs: This has been our primary usage of Promtail in the past - gathering pods logs and host logs (to include k8s audit logs, controlplane logs, etc). +1. Provide a tool that has numerous export options to cover specific needs for environments: Current known requirements include Loki, S3, and SIEM tools like Elastic and Splunk. Ideally the tool of choice supports all of these and more, allowing for expansion as new environments require it. +1. Choose a tool that does not require major changes in our logging stack, but is flexible for future adjustments to the stack: As we do have active users of our product we want to be careful in switching tools, so ideally we would like a tool that is a "drop-in" replacement. However, we don't want to rule out future changes to other pieces of the stack (i.e. Loki) so choosing a tool that doesn't lock us into Loki is important. +1. Focus on the log collection/shipping problem: While there are a number of tools that offer far more than just logging pipelines (metrics, traces, etc), we don't currently see a need to focus on these tools. These features are seen as a nice to have, but not being evaluated as the focus here. + +Three tools in the space of log collection were considered: +1. [Vector](https://vector.dev/): Opensource and maintained by Datadog, Vector provides input integrations with Kubernetes logs, arbitrary files, and [other sources](https://vector.dev/docs/reference/configuration/sources/). It has the necessary export integrations with Loki, S3, Elastic, Splunk and a [number of other sinks](https://vector.dev/docs/reference/configuration/sinks/). Vector is a newer tool that has not yet reached a 1.0 release, but has risen in popularity due to its performance improvements over other tools. +1. [FluentBit](https://fluentbit.io/): Fluentbit was historically used in Big Bang and supports file based inputs as well as [other inputs](https://docs.fluentbit.io/manual/pipeline/inputs). It also supports the necessary output integrations (Loki, S3, Elastic, Splunk and [others](https://docs.fluentbit.io/manual/pipeline/outputs)). FluentBit is a CNCF graduated project and is relatively mature. Fluentbit fell out of favor with Big Bang due to some of the complexities around managing it at scale, specifically with its buffering. +1. [Grafana Alloy](https://grafana.com/docs/alloy/latest/): Alloy is a distribution of the OpenTelemetry Collector, opensource and maintained by Grafana Labs. It supports the necessary [inputs and outputs](https://grafana.com/docs/alloy/latest/reference/components/) (local file/k8s logs, Loki and S3). As a distribution of OTel it supports vendor-agnostic output formats and can be integrated with numerous other tools through the OTel ecosystem. While Alloy itself is relatively new, it is built on the previous codebase of Grafana Agent and the existing OTel framework. Notably it does not have any direct integrations with Splunk or Elastic, and its S3 integration is noted as experimental. + +### Decision and Impact + +Vector has been chosen as our replacement for Promtail. Primary motivations include: +1. Vector has an extensive "component" catalog for inputs and outputs, with complete coverage of all currently desired export locations (and all are noted as "stable" integrations). +1. Vector's configuration is simple and works well in helm/with UDS helm overrides (easy to add additional export locations via bundle overrides for example). +1. Despite being a newer project, Vector's community is very active - with the most active contributors and GitHub stars compared to the other two tools. +1. Vector is [significantly more performant](https://github.com/vectordotdev/vector?tab=readme-ov-file#performance) than other tooling in the space on most categories of metrics. + +As with any decisions of tooling in core this can always be reevaluated in the future as different tools or factors affect how we look at our logging stack. diff --git a/src/vector/chart/templates/uds-package.yaml b/src/vector/chart/templates/uds-package.yaml index 6cece0d01..fa3bfa6f8 100644 --- a/src/vector/chart/templates/uds-package.yaml +++ b/src/vector/chart/templates/uds-package.yaml @@ -6,6 +6,15 @@ metadata: spec: network: allow: + - direction: Ingress + selector: + app.kubernetes.io/name: vector + remoteNamespace: monitoring + remoteSelector: + app.kubernetes.io/name: prometheus + port: 9090 + description: "Prometheus Metrics" + - direction: Egress selector: app.kubernetes.io/name: vector diff --git a/src/vector/pepr.log b/src/vector/pepr.log deleted file mode 100644 index ec479815b..000000000 --- a/src/vector/pepr.log +++ /dev/null @@ -1,102 +0,0 @@ -watcher {"level":20,"time":1725043340959,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /api/v1/pods."} -watcher {"level":20,"time":1725043340960,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /api/v1/services."} -watcher {"level":20,"time":1725043340960,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /apis/discovery.k8s.io/v1/endpointslices."} -watcher {"level":20,"time":1725043340961,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /apis/uds.dev/v1alpha1/packages."} -watcher {"level":20,"time":1725043340961,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /apis/uds.dev/v1alpha1/packages."} -watcher {"level":20,"time":1725043340963,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Resetting pending promise and dequeuing"} -watcher {"level":20,"time":1725043340963,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Reconciling kubernetes"} -watcher {"level":20,"time":1725043340963,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","component":"operator.generators","msg":"Processing watch for api service, getting endpoint slices for updating API server CIDR"} -watcher {"level":20,"time":1725043340968,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Resetting pending promise and dequeuing"} -watcher {"level":20,"time":1725043340968,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"No element, not dequeuing"} -watcher {"level":20,"time":1725043343189,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","apiVersion":"pepr.dev/v1","data":{"__pepr_do_not_delete__":"k-thx-bye"},"kind":"PeprStore","metadata":{"creationTimestamp":"2024-08-30T18:42:20Z","generation":1,"managedFields":[{"apiVersion":"pepr.dev/v1","fieldsType":"FieldsV1","fieldsV1":{"f:data":{"f:__pepr_do_not_delete__":{}}},"manager":"pepr","operation":"Apply","time":"2024-08-30T18:42:20Z"}],"name":"pepr-uds-core-schedule","namespace":"pepr-system","resourceVersion":"1486","uid":"f1f24a26-532e-4873-b901-bf768f0c16ce"},"msg":"Pepr Store migration"} -watcher {"level":50,"time":1725043343190,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","err":{"type":"Error","message":"No operations specified","stack":"Error: No operations specified\n at Object.Patch (/app/node_modules/kubernetes-fluent-client/dist/fluent/index.js:195:19)\n at flushCache (/app/node_modules/pepr/dist/lib.js:735:101)\n at #migrateAndSetupWatch (/app/node_modules/pepr/dist/lib.js:777:11)\n at /app/node_modules/pepr/dist/lib.js:716:159\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)"},"msg":"Pepr store update failure"} -watcher {"level":20,"time":1725043343198,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","apiVersion":"pepr.dev/v1","data":{"__pepr_do_not_delete__":"k-thx-bye"},"kind":"PeprStore","metadata":{"creationTimestamp":"2024-08-30T18:42:20Z","generation":1,"managedFields":[{"apiVersion":"pepr.dev/v1","fieldsType":"FieldsV1","fieldsV1":{"f:data":{"f:__pepr_do_not_delete__":{}}},"manager":"pepr","operation":"Apply","time":"2024-08-30T18:42:20Z"}],"name":"pepr-uds-core-schedule","namespace":"pepr-system","resourceVersion":"1486","uid":"f1f24a26-532e-4873-b901-bf768f0c16ce"},"msg":"Pepr Store update"} -watcher {"level":30,"time":1725043343198,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"✅ Scheduling processed"} -watcher {"level":30,"time":1725043353257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"2 ms"} -watcher {"level":30,"time":1725043353259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":20,"time":1725043357109,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","apiVersion":"uds.dev/v1alpha1","kind":"Package","metadata":{"annotations":{"meta.helm.sh/release-name":"uds-metrics-server-config","meta.helm.sh/release-namespace":"metrics-server"},"creationTimestamp":"2024-08-30T18:42:37Z","generation":1,"labels":{"app.kubernetes.io/managed-by":"Helm"},"managedFields":[{"apiVersion":"uds.dev/v1alpha1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app.kubernetes.io/managed-by":{}}},"f:spec":{".":{},"f:network":{".":{},"f:allow":{}}}},"manager":"uds","operation":"Update","time":"2024-08-30T18:42:37Z"}],"name":"metrics-server","namespace":"metrics-server","resourceVersion":"1541","uid":"02935348-664a-40f6-a88a-e6cb997e2c52"},"spec":{"network":{"allow":[{"direction":"Egress","port":10250,"remoteGenerated":"Anywhere","selector":{"app.kubernetes.io/name":"metrics-server"}},{"direction":"Egress","remoteGenerated":"KubeAPI","selector":{"app.kubernetes.io/name":"metrics-server"}},{"direction":"Ingress","port":10250,"remoteGenerated":"Anywhere","selector":{"app.kubernetes.io/name":"metrics-server"}}]}},"msg":"Watch event ADDED received"} -watcher {"level":20,"time":1725043357110,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","apiVersion":"uds.dev/v1alpha1","kind":"Package","metadata":{"annotations":{"meta.helm.sh/release-name":"uds-metrics-server-config","meta.helm.sh/release-namespace":"metrics-server"},"creationTimestamp":"2024-08-30T18:42:37Z","generation":1,"labels":{"app.kubernetes.io/managed-by":"Helm"},"managedFields":[{"apiVersion":"uds.dev/v1alpha1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app.kubernetes.io/managed-by":{}}},"f:spec":{".":{},"f:network":{".":{},"f:allow":{}}}},"manager":"uds","operation":"Update","time":"2024-08-30T18:42:37Z"}],"name":"metrics-server","namespace":"metrics-server","resourceVersion":"1541","uid":"02935348-664a-40f6-a88a-e6cb997e2c52"},"spec":{"network":{"allow":[{"direction":"Egress","port":10250,"remoteGenerated":"Anywhere","selector":{"app.kubernetes.io/name":"metrics-server"}},{"direction":"Egress","remoteGenerated":"KubeAPI","selector":{"app.kubernetes.io/name":"metrics-server"}},{"direction":"Ingress","port":10250,"remoteGenerated":"Anywhere","selector":{"app.kubernetes.io/name":"metrics-server"}}]}},"msg":"Watch event ADDED received"} -watcher {"level":20,"time":1725043357110,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Enqueueing metrics-server/metrics-server"} -watcher {"level":20,"time":1725043357110,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Reconciling metrics-server"} -watcher {"level":20,"time":1725043357110,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Resetting pending promise and dequeuing"} -watcher {"level":20,"time":1725043357110,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"No element, not dequeuing"} -watcher {"level":30,"time":1725043363256,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043363258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043373262,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"2 ms"} -watcher {"level":30,"time":1725043373263,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043383257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043383258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043393256,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043393257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043403258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043403259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043413263,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043413265,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043423258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043423258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043433259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043433259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043443259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043443260,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043453258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043453258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043463260,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043463262,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043473259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043473259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043483262,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043483264,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043493258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043493258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043503257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043503258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043513257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043513259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043523258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043523259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043533256,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043533256,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043543258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043543259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043553264,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043553265,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043563258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043563259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043573258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043573258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043583257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043583258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043593261,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043593261,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043603258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043603260,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043613261,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043613262,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043623256,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043623257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043633257,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043633259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043643259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043643259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043653260,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043653260,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":20,"time":1725043659051,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event reconnect received. Reconnecting after 1 attempt."} -watcher {"level":20,"time":1725043659060,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event list received. {\n \"apiVersion\": \"uds.dev/v1alpha1\",\n \"items\": [\n {\n \"apiVersion\": \"uds.dev/v1alpha1\",\n \"kind\": \"Package\",\n \"metadata\": {\n \"annotations\": {\n \"meta.helm.sh/release-name\": \"uds-metrics-server-config\",\n \"meta.helm.sh/release-namespace\": \"metrics-server\"\n },\n \"creationTimestamp\": \"2024-08-30T18:42:37Z\",\n \"generation\": 1,\n \"labels\": {\n \"app.kubernetes.io/managed-by\": \"Helm\"\n },\n \"managedFields\": [\n {\n \"apiVersion\": \"uds.dev/v1alpha1\",\n \"fieldsType\": \"FieldsV1\",\n \"fieldsV1\": {\n \"f:metadata\": {\n \"f:annotations\": {\n \".\": {},\n \"f:meta.helm.sh/release-name\": {},\n \"f:meta.helm.sh/release-namespace\": {}\n },\n \"f:labels\": {\n \".\": {},\n \"f:app.kubernetes.io/managed-by\": {}\n }\n },\n \"f:spec\": {\n \".\": {},\n \"f:network\": {\n \".\": {},\n \"f:allow\": {}\n }\n }\n },\n \"manager\": \"uds\",\n \"operation\": \"Update\",\n \"time\": \"2024-08-30T18:42:37Z\"\n }\n ],\n \"name\": \"metrics-server\",\n \"namespace\": \"metrics-server\",\n \"resourceVersion\": \"1541\",\n \"uid\": \"02935348-664a-40f6-a88a-e6cb997e2c52\"\n },\n \"spec\": {\n \"network\": {\n \"allow\": [\n {\n \"direction\": \"Egress\",\n \"port\": 10250,\n \"remoteGenerated\": \"Anywhere\",\n \"selector\": {\n \"app.kubernetes.io/name\": \"metrics-server\"\n }\n },\n {\n \"direction\": \"Egress\",\n \"remoteGenerated\": \"KubeAPI\",\n \"selector\": {\n \"app.kubernetes.io/name\": \"metrics-server\"\n }\n },\n {\n \"direction\": \"Ingress\",\n \"port\": 10250,\n \"remoteGenerated\": \"Anywhere\",\n \"selector\": {\n \"app.kubernetes.io/name\": \"metrics-server\"\n }\n }\n ]\n }\n }\n }\n ],\n \"kind\": \"PackageList\",\n \"metadata\": {\n \"continue\": \"\",\n \"resourceVersion\": \"1886\"\n }\n}."} -watcher {"level":20,"time":1725043659063,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /apis/uds.dev/v1alpha1/packages."} -watcher {"level":20,"time":1725043660574,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event reconnect received. Reconnecting after 1 attempt."} -watcher {"level":20,"time":1725043660582,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event list received. {\n \"apiVersion\": \"uds.dev/v1alpha1\",\n \"items\": [\n {\n \"apiVersion\": \"uds.dev/v1alpha1\",\n \"kind\": \"Package\",\n \"metadata\": {\n \"annotations\": {\n \"meta.helm.sh/release-name\": \"uds-metrics-server-config\",\n \"meta.helm.sh/release-namespace\": \"metrics-server\"\n },\n \"creationTimestamp\": \"2024-08-30T18:42:37Z\",\n \"generation\": 1,\n \"labels\": {\n \"app.kubernetes.io/managed-by\": \"Helm\"\n },\n \"managedFields\": [\n {\n \"apiVersion\": \"uds.dev/v1alpha1\",\n \"fieldsType\": \"FieldsV1\",\n \"fieldsV1\": {\n \"f:metadata\": {\n \"f:annotations\": {\n \".\": {},\n \"f:meta.helm.sh/release-name\": {},\n \"f:meta.helm.sh/release-namespace\": {}\n },\n \"f:labels\": {\n \".\": {},\n \"f:app.kubernetes.io/managed-by\": {}\n }\n },\n \"f:spec\": {\n \".\": {},\n \"f:network\": {\n \".\": {},\n \"f:allow\": {}\n }\n }\n },\n \"manager\": \"uds\",\n \"operation\": \"Update\",\n \"time\": \"2024-08-30T18:42:37Z\"\n }\n ],\n \"name\": \"metrics-server\",\n \"namespace\": \"metrics-server\",\n \"resourceVersion\": \"1541\",\n \"uid\": \"02935348-664a-40f6-a88a-e6cb997e2c52\"\n },\n \"spec\": {\n \"network\": {\n \"allow\": [\n {\n \"direction\": \"Egress\",\n \"port\": 10250,\n \"remoteGenerated\": \"Anywhere\",\n \"selector\": {\n \"app.kubernetes.io/name\": \"metrics-server\"\n }\n },\n {\n \"direction\": \"Egress\",\n \"remoteGenerated\": \"KubeAPI\",\n \"selector\": {\n \"app.kubernetes.io/name\": \"metrics-server\"\n }\n },\n {\n \"direction\": \"Ingress\",\n \"port\": 10250,\n \"remoteGenerated\": \"Anywhere\",\n \"selector\": {\n \"app.kubernetes.io/name\": \"metrics-server\"\n }\n }\n ]\n }\n }\n }\n ],\n \"kind\": \"PackageList\",\n \"metadata\": {\n \"continue\": \"\",\n \"resourceVersion\": \"1890\"\n }\n}."} -watcher {"level":20,"time":1725043660584,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","msg":"Watch event connect received. /apis/uds.dev/v1alpha1/packages."} -watcher {"level":30,"time":1725043663258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043663259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043673259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043673259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043683262,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043683265,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"1 ms"} -watcher {"level":30,"time":1725043693258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043693258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043703258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043703258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043713258,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043713259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043723259,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} -watcher {"level":30,"time":1725043723260,"pid":17,"hostname":"pepr-uds-core-watcher-56b6d67564-45ch8","method":"GET","url":"/healthz","status":200,"duration":"0 ms"} From 376b108a71d7f42feb5cf4b299bee00264055b6c Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 4 Sep 2024 12:20:09 -0600 Subject: [PATCH 04/17] chore: use pod name if available --- src/vector/values/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/vector/values/values.yaml b/src/vector/values/values.yaml index 769a34687..6dd47a0bb 100644 --- a/src/vector/values/values.yaml +++ b/src/vector/values/values.yaml @@ -26,6 +26,8 @@ customConfig: if !exists(.kubernetes.pod_labels.app) { if exists(.kubernetes.pod_labels."app.kubernetes.io/name") { .kubernetes.pod_labels.app = .kubernetes.pod_labels."app.kubernetes.io/name" + } else if exists(.kubernetes.pod_labels.name) { + .kubernetes.pod_labels.app = .kubernetes.pod_labels.name } else if exists(.kubernetes.pod_owner) { .kubernetes.pod_labels.app = .kubernetes.pod_owner } else { From 3bb15ea7f2d04fb2375970f73a16d6f8dbab9415 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 4 Sep 2024 12:29:02 -0600 Subject: [PATCH 05/17] chore: netpols --- src/vector/chart/templates/uds-package.yaml | 16 ++++++++++++++++ src/vector/chart/values.yaml | 16 ++++++++++++++++ src/vector/tasks.yaml | 2 +- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/vector/chart/templates/uds-package.yaml b/src/vector/chart/templates/uds-package.yaml index fa3bfa6f8..b6bf5bbc1 100644 --- a/src/vector/chart/templates/uds-package.yaml +++ b/src/vector/chart/templates/uds-package.yaml @@ -28,3 +28,19 @@ spec: app.kubernetes.io/name: loki port: 8080 description: "Write Logs to Loki" + + # Custom rules for additional networking access + {{- range .Values.additionalNetworkAllow }} + - direction: {{ .direction }} + selector: + {{ .selector | toYaml | nindent 10 }} + {{- if not .remoteGenerated }} + remoteNamespace: {{ .remoteNamespace }} + remoteSelector: + {{ .remoteSelector | toYaml | nindent 10 }} + port: {{ .port }} + {{- else }} + remoteGenerated: {{ .remoteGenerated }} + {{- end }} + description: {{ .description }} + {{- end }} diff --git a/src/vector/chart/values.yaml b/src/vector/chart/values.yaml index e69de29bb..f2d4d867f 100644 --- a/src/vector/chart/values.yaml +++ b/src/vector/chart/values.yaml @@ -0,0 +1,16 @@ +additionalNetworkAllow: [] +# Examples: +# - direction: Egress +# selector: +# app.kubernetes.io/name: vector +# remoteNamespace: elastic +# remoteSelector: +# app.kubernetes.io/name: elastic +# port: 9090 +# description: "Elastic Storage" +# - direction: Egress +# selector: +# app.kubernetes.io/name: vector +# remoteGenerated: Anywhere +# port: 80 +# description: "S3 Storage" diff --git a/src/vector/tasks.yaml b/src/vector/tasks.yaml index 3a71a4295..69dfbf4ff 100644 --- a/src/vector/tasks.yaml +++ b/src/vector/tasks.yaml @@ -5,6 +5,6 @@ tasks: wait: cluster: kind: Pod - name: app.kubernetes.io/instance=vector + name: app.kubernetes.io/name=vector namespace: vector condition: Ready From 1ff13e5fbf014bda36d18a197216179b3379f794 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 4 Sep 2024 12:42:45 -0600 Subject: [PATCH 06/17] fix: yaml lint --- pkg/sumdb/sum.golang.org/latest | 5 +++++ src/vector/values/values.yaml | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 pkg/sumdb/sum.golang.org/latest diff --git a/pkg/sumdb/sum.golang.org/latest b/pkg/sumdb/sum.golang.org/latest new file mode 100644 index 000000000..f1ed86ecf --- /dev/null +++ b/pkg/sumdb/sum.golang.org/latest @@ -0,0 +1,5 @@ +go.sum database tree +29415271 +DxkqmXb+ZJUZX47b/+QZY/fsiKmormi9RSzzKf1XHx0= + +— sum.golang.org Az3grs1hmZ3YePBZ5uxV7KkeDUtjV3fnBJasM+i5Qj+2ndhhtha7GJSrnBMaK11JHP3ETov0jQUfWEeu5r40EI3u0w0= diff --git a/src/vector/values/values.yaml b/src/vector/values/values.yaml index 6dd47a0bb..2a60652aa 100644 --- a/src/vector/values/values.yaml +++ b/src/vector/values/values.yaml @@ -6,15 +6,15 @@ customConfig: sources: pod_logs: type: "kubernetes_logs" - oldest_first: true + oldest_first: true node_logs: type: "file" include: ["/var/log/*"] - oldest_first: true + oldest_first: true k8s_logs: type: "file" include: ["/var/log/kubernetes/**/*.log"] - oldest_first: true + oldest_first: true internal_metrics: type: internal_metrics From abb01b74b800f00c0e10cffc9fdd9b551c2d2610 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 4 Sep 2024 14:46:31 -0600 Subject: [PATCH 07/17] chore: simpler [ci skip] --- src/vector/values/values.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/vector/values/values.yaml b/src/vector/values/values.yaml index 2a60652aa..8351abbd6 100644 --- a/src/vector/values/values.yaml +++ b/src/vector/values/values.yaml @@ -7,14 +7,12 @@ customConfig: pod_logs: type: "kubernetes_logs" oldest_first: true + ignore_older_secs: 900 # Reduce duplication across initial upgrade node_logs: type: "file" - include: ["/var/log/*"] - oldest_first: true - k8s_logs: - type: "file" - include: ["/var/log/kubernetes/**/*.log"] + include: ["/var/log/*", "/var/log/kubernetes/**/*.log"] oldest_first: true + ignore_older_secs: 900 # Reduce duplication across initial upgrade internal_metrics: type: internal_metrics @@ -34,9 +32,9 @@ customConfig: .kubernetes.pod_labels.app = .kubernetes.pod_name } } - host_logs_labelled: + node_logs_labelled: type: remap - inputs: ["node_logs", "k8s_logs"] + inputs: ["node_logs"] source: | .node_name = "${NODE_HOSTNAME}" if contains(string!(.file), "/var/log/kubernetes/") { @@ -65,7 +63,7 @@ customConfig: max_size: 1073741824 # 1GiB loki_host: type: "loki" - inputs: ["host_logs_labelled"] + inputs: ["node_logs_labelled"] endpoint: "http://loki-gateway.loki.svc.cluster.local:80" path: "/loki/api/v1/push" encoding: From f5c5bec261e64b932b84604a5dd42a361e1d03ac Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 5 Sep 2024 08:22:37 -0600 Subject: [PATCH 08/17] chore: add wip IB image [ci skip] --- src/vector/Dockerfile | 15 +++++++++++++++ src/vector/values/registry1-values.yaml | 4 ++-- src/vector/zarf.yaml | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 src/vector/Dockerfile diff --git a/src/vector/Dockerfile b/src/vector/Dockerfile new file mode 100644 index 000000000..566b2457d --- /dev/null +++ b/src/vector/Dockerfile @@ -0,0 +1,15 @@ +ARG BASE_REGISTRY=registry1.dso.mil +ARG BASE_IMAGE=ironbank/google/distroless/static +ARG BASE_TAG=nonroot + +FROM docker.io/timberio/vector:0.40.1 AS upstream + +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} + +COPY --from=upstream /usr/local/bin/* /usr/local/bin/ +COPY --from=upstream /var/lib/vector /var/lib/vector +COPY --from=upstream /etc/vector/vector.yaml /etc/vector/vector.yaml + +HEALTHCHECK NONE + +ENTRYPOINT ["/usr/local/bin/vector"] diff --git a/src/vector/values/registry1-values.yaml b/src/vector/values/registry1-values.yaml index 9ff27eecd..c31b27984 100644 --- a/src/vector/values/registry1-values.yaml +++ b/src/vector/values/registry1-values.yaml @@ -1,3 +1,3 @@ image: - repository: timberio/vector - tag: 0.40.1-distroless-static + repository: docker.io/mjnagel/vector-ib + tag: 0.40.1 diff --git a/src/vector/zarf.yaml b/src/vector/zarf.yaml index b661d6c01..6a722d0d5 100644 --- a/src/vector/zarf.yaml +++ b/src/vector/zarf.yaml @@ -31,7 +31,7 @@ components: valuesFiles: - values/registry1-values.yaml images: - - timberio/vector:0.40.1-distroless-static # registry1 image is WIP + - docker.io/mjnagel/vector-ib:0.40.1 - name: vector required: true From eb91b39ce953bf4c55c494479116b4a63cd3ba48 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 5 Sep 2024 08:23:06 -0600 Subject: [PATCH 09/17] chore: revert age filter [ci skip] --- src/vector/values/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/vector/values/values.yaml b/src/vector/values/values.yaml index 8351abbd6..4e712d8df 100644 --- a/src/vector/values/values.yaml +++ b/src/vector/values/values.yaml @@ -7,12 +7,10 @@ customConfig: pod_logs: type: "kubernetes_logs" oldest_first: true - ignore_older_secs: 900 # Reduce duplication across initial upgrade node_logs: type: "file" include: ["/var/log/*", "/var/log/kubernetes/**/*.log"] oldest_first: true - ignore_older_secs: 900 # Reduce duplication across initial upgrade internal_metrics: type: internal_metrics From 81bc8647ce3d901f77f3ab0d0201a0bb54eb8cef Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 5 Sep 2024 13:33:25 -0600 Subject: [PATCH 10/17] cleanup [ci skip] --- pkg/sumdb/sum.golang.org/latest | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 pkg/sumdb/sum.golang.org/latest diff --git a/pkg/sumdb/sum.golang.org/latest b/pkg/sumdb/sum.golang.org/latest deleted file mode 100644 index f1ed86ecf..000000000 --- a/pkg/sumdb/sum.golang.org/latest +++ /dev/null @@ -1,5 +0,0 @@ -go.sum database tree -29415271 -DxkqmXb+ZJUZX47b/+QZY/fsiKmormi9RSzzKf1XHx0= - -— sum.golang.org Az3grs1hmZ3YePBZ5uxV7KkeDUtjV3fnBJasM+i5Qj+2ndhhtha7GJSrnBMaK11JHP3ETov0jQUfWEeu5r40EI3u0w0= From 2e6a796fac0cfe897a46d8d72a9dd1234c8e334d Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 5 Sep 2024 16:39:44 -0600 Subject: [PATCH 11/17] docs: update to vector capabilities [ci skip] --- docs/application-baseline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/application-baseline.md b/docs/application-baseline.md index d2410c243..e9f7f02fb 100644 --- a/docs/application-baseline.md +++ b/docs/application-baseline.md @@ -18,7 +18,7 @@ For optimal deployment and operational efficiency, it is important to deliver a | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | **Service Mesh** | **[Istio](https://istio.io/):** A powerful service mesh that provides traffic management, load balancing, security, and observability features. | | **Monitoring** | **[Metrics Server](https://kubernetes-sigs.github.io/metrics-server/):** Provides container resource utilization metrics API for Kubernetes clusters. Metrics server is an optional (non-default) component since most Kubernetes distros provide it by default.

**[Prometheus](https://prometheus.io/):** Scrapes Metrics Server API and application metrics and stores the data in a time-series database for insights into application health and performance.

**[Grafana](https://grafana.com/grafana/):** Provides visualization and alerting capabilities based on Prometheus's time-series database of metrics. | -| **Logging** | **[Vector](https://vector.dev/):** A companion agent that efficiently gathers and sends container logs to Loki, simplifying log monitoring, troubleshooting, and compliance auditing, enhancing the overall observability of the mission environment.

**[Loki](https://grafana.com/docs/loki/latest/):** A log aggregation system that allows users to store, search, and analyze logs across their applications. | +| **Logging** | **[Vector](https://vector.dev/):** A companion agent that efficiently gathers and sends container logs to Loki and other storage locations (S3, SIEM tools, etc), simplifying log monitoring, troubleshooting, and compliance auditing, enhancing the overall observability of the mission environment.

**[Loki](https://grafana.com/docs/loki/latest/):** A log aggregation system that allows users to store, search, and analyze logs across their applications. | | **Security and Compliance** | **[NeuVector](https://open-docs.neuvector.com/):** Offers container-native security, protecting applications against threats and vulnerabilities.

**[Pepr](https://pepr.dev/):** UDS policy engine and operator for enhanced security and compliance.| | **Identity and Access Management** | **[Keycloak](https://www.keycloak.org/):** A robust open-source Identity and Access Management solution, providing centralized authentication, authorization, and user management for enhanced security and control over access to mission-critical resources.| | **Backup and Restore** | **[Velero](https://velero.io/):** Provides backup and restore capabilities for Kubernetes clusters, ensuring data protection and disaster recovery.| From 3f01146e51aefca386f36dabca8247e2a65a926b Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Mon, 9 Sep 2024 14:53:02 -0600 Subject: [PATCH 12/17] chore: oscal update --- src/istio/oscal-component.yaml | 2 +- src/vector/oscal-component.yaml | 17 +++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/istio/oscal-component.yaml b/src/istio/oscal-component.yaml index f3c4b65a5..d95b5c4ab 100644 --- a/src/istio/oscal-component.yaml +++ b/src/istio/oscal-component.yaml @@ -539,7 +539,7 @@ component-definition: # Expected values expected_istiod_port := 15012 expected_istiod_protocol := "TCP" - required_namespaces := {"authservice", "grafana", "keycloak", "loki", "metrics-server", "monitoring", "neuvector", "promtail", "velero"} + required_namespaces := {"authservice", "grafana", "keycloak", "loki", "metrics-server", "monitoring", "neuvector", "vector", "velero"} # Validate NetworkPolicy for Istiod in required namespaces validate { diff --git a/src/vector/oscal-component.yaml b/src/vector/oscal-component.yaml index 1a5bb3497..fef87cc00 100644 --- a/src/vector/oscal-component.yaml +++ b/src/vector/oscal-component.yaml @@ -58,7 +58,7 @@ component-definition: # Control Implementation Logging daemons are present on each node that BigBang is installed on. Out of the box, the following events are captured: - * all containers emitting to STDOUT or STDERR (captured by container runtime translating container logs to /var/log/containers). + * all containers emitting to STDOUT or STDERR (captured by container runtime creating containers logs under /var/log/pods). * all kubernetes api server requests. * all events emitted by the kubelet. remarks: This control is fully implemented by this tool. @@ -78,9 +78,7 @@ component-definition: Event outcomes can include indicators of event success or failure and event-specific results (e.g., the security state of the information system after the event occurred). # Control Implementation - Logs are captured by vector from the node. The node logs will contain the necessary log data from all pods/applications inside the selected nodes. - Validating `logfmt` as the config.logFormat would be the goal. This is currently a secret mounted to /etc/vector/vector.yaml in the vector container. We will ensure the vector.yaml file is at a minimum the target config. - https://grafana.com/docs/loki/latest/send-data/vector/stages/logfmt/ + Logs are captured by vector from the node. The node logs will contain the necessary log data from all pods/applications inside the selected nodes as well as Kubernetes audit logs. remarks: This control is fully implemented by this tool. links: - href: "#98b97ec9-a9ce-4444-83d8-71066270a424" @@ -105,8 +103,6 @@ component-definition: * time of the event (UTC). * source of event (pod, namespace, container id). Applications are responsible for providing all other information. - Validating `logfmt` as the config.logFormat would be the goal. This is currently a secret mounted to /etc/vector/vector.yaml in the vector container. We will ensure the vector.yaml file is at a minimum the target config. - https://grafana.com/docs/loki/latest/send-data/vector/stages/logfmt/ remarks: This control is fully implemented by this tool. links: - href: "#98b97ec9-a9ce-4444-83d8-71066270a424" @@ -124,11 +120,7 @@ component-definition: - uuid: D552C935-E40C-4A03-B5CC-4605EBD95B6D title: Vector rlinks: - - href: https://grafana.com/docs/loki/latest/clients/vector/ - - uuid: 211C474B-E11A-4DD2-8075-50CDAC507CDC - title: Big Bang Vector package - rlinks: - - href: https://repo1.dso.mil/platform-one/big-bang/apps/sandbox/vector + - href: https://vector.dev/ - uuid: 98b97ec9-a9ce-4444-83d8-71066270a424 title: Lula Validation rlinks: @@ -263,6 +255,7 @@ component-definition: some container in containers container.name == "vector" some i - container.args[i] == "-config.file=/etc/vector/vector.yaml" + container.args[i] == "--config-dir" + container.args[i] == "/etc/vector/" } } From 42c8e3b1baf58ba949ad01a94689e2fc70d81af1 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 11 Sep 2024 11:12:17 -0600 Subject: [PATCH 13/17] chore: helm chart update [skip ci] --- src/vector/common/zarf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vector/common/zarf.yaml b/src/vector/common/zarf.yaml index 65734e2b4..23b2efad0 100644 --- a/src/vector/common/zarf.yaml +++ b/src/vector/common/zarf.yaml @@ -14,7 +14,7 @@ components: localPath: ../chart - name: vector url: https://helm.vector.dev - version: 0.35.0 + version: 0.36.0 namespace: vector gitPath: charts/vector valuesFiles: From 7020145d56824ecd86a480b1107452d01d7df0cd Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Mon, 16 Sep 2024 15:01:27 -0600 Subject: [PATCH 14/17] chore: updates --- src/pepr/zarf.yaml | 43 +++++++++---------------- src/vector/README.md | 4 +++ src/vector/common/zarf.yaml | 2 +- src/vector/values/registry1-values.yaml | 2 +- src/vector/values/unicorn-values.yaml | 2 +- src/vector/values/upstream-values.yaml | 2 +- src/vector/values/values.yaml | 12 +++++++ src/vector/zarf.yaml | 6 ++-- 8 files changed, 38 insertions(+), 35 deletions(-) diff --git a/src/pepr/zarf.yaml b/src/pepr/zarf.yaml index 5dafad221..25c0e2f2a 100644 --- a/src/pepr/zarf.yaml +++ b/src/pepr/zarf.yaml @@ -52,31 +52,18 @@ components: actions: onDeploy: before: - - cmd: ./zarf tools kubectl annotate secret -n pepr-system pepr-uds-core-api-token meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate secret -n pepr-system pepr-uds-core-module meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate secret -n pepr-system pepr-uds-core-tls meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate serviceaccount -n pepr-system pepr-uds-core meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate clusterrolebinding pepr-uds-core meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate clusterrole pepr-uds-core meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate role -n pepr-system pepr-uds-core-store meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate rolebinding -n pepr-system pepr-uds-core-store meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate service -n pepr-system pepr-uds-core meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate service -n pepr-system pepr-uds-core-watcher meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate deployment -n pepr-system pepr-uds-core meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate deployment -n pepr-system pepr-uds-core-watcher meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate mutatingwebhookconfiguration -n pepr-system pepr-uds-core meta.helm.sh/release-name=module --overwrite || true - mute: true - - cmd: ./zarf tools kubectl annotate validatingwebhookconfiguration -n pepr-system pepr-uds-core meta.helm.sh/release-name=module --overwrite || true - mute: true + - cmd: | + ./zarf tools kubectl annotate secret -n pepr-system pepr-uds-core-api-token meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate secret -n pepr-system pepr-uds-core-module meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate secret -n pepr-system pepr-uds-core-tls meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate serviceaccount -n pepr-system pepr-uds-core meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate clusterrolebinding pepr-uds-core meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate clusterrole pepr-uds-core meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate role -n pepr-system pepr-uds-core-store meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate rolebinding -n pepr-system pepr-uds-core-store meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate service -n pepr-system pepr-uds-core meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate service -n pepr-system pepr-uds-core-watcher meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate deployment -n pepr-system pepr-uds-core meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate deployment -n pepr-system pepr-uds-core-watcher meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate mutatingwebhookconfiguration -n pepr-system pepr-uds-core meta.helm.sh/release-name=module --overwrite || true + ./zarf tools kubectl annotate validatingwebhookconfiguration -n pepr-system pepr-uds-core meta.helm.sh/release-name=module --overwrite || true diff --git a/src/vector/README.md b/src/vector/README.md index f40cacb2e..c2424a688 100644 --- a/src/vector/README.md +++ b/src/vector/README.md @@ -34,3 +34,7 @@ Vector has been chosen as our replacement for Promtail. Primary motivations incl 1. Vector is [significantly more performant](https://github.com/vectordotdev/vector?tab=readme-ov-file#performance) than other tooling in the space on most categories of metrics. As with any decisions of tooling in core this can always be reevaluated in the future as different tools or factors affect how we look at our logging stack. + +### Upgrade Considerations + +During the upgrade there may be some duplication/overlap of log lines shipped to Loki due to the transition from Promtail's "position" file to Vector's "checkpoint" file (both used for tracking the last log line scraped/shipped). Grafana provides a built in feature to de-duplicate log entries when querying Loki. diff --git a/src/vector/common/zarf.yaml b/src/vector/common/zarf.yaml index 23b2efad0..b020db0e8 100644 --- a/src/vector/common/zarf.yaml +++ b/src/vector/common/zarf.yaml @@ -14,7 +14,7 @@ components: localPath: ../chart - name: vector url: https://helm.vector.dev - version: 0.36.0 + version: 0.36.1 namespace: vector gitPath: charts/vector valuesFiles: diff --git a/src/vector/values/registry1-values.yaml b/src/vector/values/registry1-values.yaml index 7e4f21195..85509e7b4 100644 --- a/src/vector/values/registry1-values.yaml +++ b/src/vector/values/registry1-values.yaml @@ -1,3 +1,3 @@ image: repository: registry1.dso.mil/ironbank/opensource/timberio/vector - tag: 0.40.2 + tag: 0.41.1 diff --git a/src/vector/values/unicorn-values.yaml b/src/vector/values/unicorn-values.yaml index 6a129b1e6..d90700602 100644 --- a/src/vector/values/unicorn-values.yaml +++ b/src/vector/values/unicorn-values.yaml @@ -1,3 +1,3 @@ image: repository: cgr.dev/du-uds-defenseunicorns/vector - tag: 0.40.1 + tag: 0.41.1 diff --git a/src/vector/values/upstream-values.yaml b/src/vector/values/upstream-values.yaml index 9fd6786d1..5180f3c7c 100644 --- a/src/vector/values/upstream-values.yaml +++ b/src/vector/values/upstream-values.yaml @@ -1,3 +1,3 @@ image: repository: timberio/vector - tag: 0.40.2-distroless-static + tag: 0.41.1-distroless-static diff --git a/src/vector/values/values.yaml b/src/vector/values/values.yaml index 4e712d8df..c8d692198 100644 --- a/src/vector/values/values.yaml +++ b/src/vector/values/values.yaml @@ -3,6 +3,9 @@ role: "Agent" customConfig: data_dir: /var/lib/vector + # Ensure e2e delivery of events + acknowledgements: + enabled: true sources: pod_logs: type: "kubernetes_logs" @@ -30,6 +33,14 @@ customConfig: .kubernetes.pod_labels.app = .kubernetes.pod_name } } + if !exists(.kubernetes.pod_labels.component) { + if exists(.kubernetes.pod_labels."app.kubernetes.io/component") { + .kubernetes.pod_labels.component = .kubernetes.pod_labels."app.kubernetes.io/component" + } else { + .kubernetes.pod_labels.component = "" + } + } + node_logs_labelled: type: remap inputs: ["node_logs"] @@ -54,6 +65,7 @@ customConfig: app: '{{`{{ kubernetes.pod_labels.app }}`}}' job: '{{`{{ kubernetes.pod_namespace }}`}}/{{`{{ kubernetes.pod_labels.app }}`}}' container: '{{`{{ kubernetes.container_name }}`}}' + component: '{{`{{ kubernetes.pod_labels.component }}`}}' host: '{{`{{ kubernetes.pod_node_name }}`}}' file: '{{`{{ file }}`}}' buffer: diff --git a/src/vector/zarf.yaml b/src/vector/zarf.yaml index 10cd7d846..738476d7d 100644 --- a/src/vector/zarf.yaml +++ b/src/vector/zarf.yaml @@ -17,7 +17,7 @@ components: valuesFiles: - values/upstream-values.yaml images: - - timberio/vector:0.40.2-distroless-static + - timberio/vector:0.41.1-distroless-static - name: vector required: true @@ -31,7 +31,7 @@ components: valuesFiles: - values/registry1-values.yaml images: - - registry1.dso.mil/ironbank/opensource/timberio/vector:0.40.2 + - registry1.dso.mil/ironbank/opensource/timberio/vector:0.41.1 - name: vector required: true @@ -45,4 +45,4 @@ components: valuesFiles: - values/unicorn-values.yaml images: - - cgr.dev/du-uds-defenseunicorns/vector:0.40.1 # todo: patch 0.40.2 update + - cgr.dev/du-uds-defenseunicorns/vector:0.41.1 From a86f4dd34f1c90da317ff2dd43a6b514a85592dc Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Mon, 16 Sep 2024 16:41:21 -0600 Subject: [PATCH 15/17] chore: docs + labels for collector --- src/vector/README.md | 4 +++- src/vector/values/values.yaml | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/vector/README.md b/src/vector/README.md index c2424a688..d22d451f7 100644 --- a/src/vector/README.md +++ b/src/vector/README.md @@ -37,4 +37,6 @@ As with any decisions of tooling in core this can always be reevaluated in the f ### Upgrade Considerations -During the upgrade there may be some duplication/overlap of log lines shipped to Loki due to the transition from Promtail's "position" file to Vector's "checkpoint" file (both used for tracking the last log line scraped/shipped). Grafana provides a built in feature to de-duplicate log entries when querying Loki. +During the upgrade there may be some duplication/overlap of log lines shipped to Loki due to the transition from Promtail's "position" file to Vector's "checkpoint" file (both used for tracking the last log line scraped/shipped). Grafana provides a built in feature to de-duplicate log entries when querying Loki, but this does not consistently work with all log lines due to the approach used for de-duplication. + +To ensure easy querying of logs across the upgrade, all logs shipped by Vector also have a `collector` label (with the value of `vector`). This can be used to filter down any logs to either what was collected by Vector or what was not collected by Vector (using the `=` and `!=` operators). In general you can use these filters to filter so that any log timestamps from before your upgrade are not collected by Vector and vice-verse post-upgrade. diff --git a/src/vector/values/values.yaml b/src/vector/values/values.yaml index c8d692198..467763b65 100644 --- a/src/vector/values/values.yaml +++ b/src/vector/values/values.yaml @@ -67,7 +67,8 @@ customConfig: container: '{{`{{ kubernetes.container_name }}`}}' component: '{{`{{ kubernetes.pod_labels.component }}`}}' host: '{{`{{ kubernetes.pod_node_name }}`}}' - file: '{{`{{ file }}`}}' + filename: '{{`{{ file }}`}}' + collector: "vector" buffer: type: disk max_size: 1073741824 # 1GiB @@ -81,7 +82,8 @@ customConfig: labels: job: '{{`{{ job }}`}}' host: '{{`{{ node_name }}`}}' - file: '{{`{{ file }}`}}' + filename: '{{`{{ file }}`}}' + collector: "vector" buffer: type: disk max_size: 1073741824 # 1GiB From 01c15003b46b19ba2e5399bfb94454b1aee87b4f Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Tue, 17 Sep 2024 12:45:57 -0600 Subject: [PATCH 16/17] fix: labelling, docs, grafana telemetry, pepr action --- src/grafana/values/values.yaml | 3 +++ src/pepr/zarf.yaml | 4 +++- src/vector/README.md | 4 ++-- src/vector/values/values.yaml | 37 +++++++++++++++++----------------- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/grafana/values/values.yaml b/src/grafana/values/values.yaml index 6ae3c9a8b..07c038ede 100644 --- a/src/grafana/values/values.yaml +++ b/src/grafana/values/values.yaml @@ -22,6 +22,9 @@ grafana.ini: reporting_enabled: false check_for_updates: false check_for_plugin_updates: false + feedback_links_enabled: false + plugins: + public_key_retrieval_disabled: true auth: # Disable the login form to force users to use SSO disable_login_form: true diff --git a/src/pepr/zarf.yaml b/src/pepr/zarf.yaml index 25c0e2f2a..904045a19 100644 --- a/src/pepr/zarf.yaml +++ b/src/pepr/zarf.yaml @@ -52,7 +52,9 @@ components: actions: onDeploy: before: - - cmd: | + - mute: true + description: "Update helm ownership for Pepr resources if necessary during the upgrade" + cmd: | ./zarf tools kubectl annotate secret -n pepr-system pepr-uds-core-api-token meta.helm.sh/release-name=module --overwrite || true ./zarf tools kubectl annotate secret -n pepr-system pepr-uds-core-module meta.helm.sh/release-name=module --overwrite || true ./zarf tools kubectl annotate secret -n pepr-system pepr-uds-core-tls meta.helm.sh/release-name=module --overwrite || true diff --git a/src/vector/README.md b/src/vector/README.md index d22d451f7..7c424cb33 100644 --- a/src/vector/README.md +++ b/src/vector/README.md @@ -37,6 +37,6 @@ As with any decisions of tooling in core this can always be reevaluated in the f ### Upgrade Considerations -During the upgrade there may be some duplication/overlap of log lines shipped to Loki due to the transition from Promtail's "position" file to Vector's "checkpoint" file (both used for tracking the last log line scraped/shipped). Grafana provides a built in feature to de-duplicate log entries when querying Loki, but this does not consistently work with all log lines due to the approach used for de-duplication. +During the upgrade there may be some duplication/overlap of log lines shipped to Loki due to the transition from Promtail's "position" file to Vector's "checkpoint" file (both used for tracking the last log line scraped/shipped). Grafana provides a built in feature to de-duplicate log entries when querying Loki, but this does not consistently work with all log lines due to the approach used by Grafana for de-duplication. -To ensure easy querying of logs across the upgrade, all logs shipped by Vector also have a `collector` label (with the value of `vector`). This can be used to filter down any logs to either what was collected by Vector or what was not collected by Vector (using the `=` and `!=` operators). In general you can use these filters to filter so that any log timestamps from before your upgrade are not collected by Vector and vice-verse post-upgrade. +To ensure easy querying of logs across the upgrade, all logs shipped by Vector also have a `collector` label (with the value of `vector`). This can be used to filter down any logs to those collected by either Vector or Promtail (using the `=` and `!=` operators). In general you can use these filters along with tracking your upgrade timing to properly ignore duplicate logs for the short upgrade period. diff --git a/src/vector/values/values.yaml b/src/vector/values/values.yaml index 467763b65..7bbe3ee60 100644 --- a/src/vector/values/values.yaml +++ b/src/vector/values/values.yaml @@ -22,23 +22,22 @@ customConfig: type: remap inputs: ["pod_logs"] source: | - if !exists(.kubernetes.pod_labels.app) { - if exists(.kubernetes.pod_labels."app.kubernetes.io/name") { - .kubernetes.pod_labels.app = .kubernetes.pod_labels."app.kubernetes.io/name" - } else if exists(.kubernetes.pod_labels.name) { - .kubernetes.pod_labels.app = .kubernetes.pod_labels.name - } else if exists(.kubernetes.pod_owner) { - .kubernetes.pod_labels.app = .kubernetes.pod_owner - } else { - .kubernetes.pod_labels.app = .kubernetes.pod_name - } + if exists(.kubernetes.pod_labels."app.kubernetes.io/name") { + .app = .kubernetes.pod_labels."app.kubernetes.io/name" + } else if exists(.kubernetes.pod_labels.app) { + .app = .kubernetes.pod_labels.app + } else if exists(.kubernetes.pod_owner) { + .app = replace!(.kubernetes.pod_owner, r'^([^/]+/)', "") + } else { + .app = .kubernetes.pod_name } - if !exists(.kubernetes.pod_labels.component) { - if exists(.kubernetes.pod_labels."app.kubernetes.io/component") { - .kubernetes.pod_labels.component = .kubernetes.pod_labels."app.kubernetes.io/component" - } else { - .kubernetes.pod_labels.component = "" - } + + if exists(.kubernetes.pod_labels."app.kubernetes.io/component") { + .component = .kubernetes.pod_labels."app.kubernetes.io/component" + } else if !exists(.kubernetes.pod_labels.component) { + .component = .kubernetes.pod_labels.component + } else { + .component = "" } node_logs_labelled: @@ -62,10 +61,10 @@ customConfig: codec: "raw_message" labels: namespace: '{{`{{ kubernetes.pod_namespace }}`}}' - app: '{{`{{ kubernetes.pod_labels.app }}`}}' - job: '{{`{{ kubernetes.pod_namespace }}`}}/{{`{{ kubernetes.pod_labels.app }}`}}' + app: '{{`{{ app }}`}}' + job: '{{`{{ kubernetes.pod_namespace }}`}}/{{`{{ app }}`}}' container: '{{`{{ kubernetes.container_name }}`}}' - component: '{{`{{ kubernetes.pod_labels.component }}`}}' + component: '{{`{{ component }}`}}' host: '{{`{{ kubernetes.pod_node_name }}`}}' filename: '{{`{{ file }}`}}' collector: "vector" From c3801a1ad4c0f1b427b8a798737ac48d2435a967 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Mon, 23 Sep 2024 08:37:34 -0600 Subject: [PATCH 17/17] chore: pr feedback --- src/vector/Dockerfile | 15 --------------- src/vector/README.md | 22 +++++++++++----------- 2 files changed, 11 insertions(+), 26 deletions(-) delete mode 100644 src/vector/Dockerfile diff --git a/src/vector/Dockerfile b/src/vector/Dockerfile deleted file mode 100644 index 566b2457d..000000000 --- a/src/vector/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -ARG BASE_REGISTRY=registry1.dso.mil -ARG BASE_IMAGE=ironbank/google/distroless/static -ARG BASE_TAG=nonroot - -FROM docker.io/timberio/vector:0.40.1 AS upstream - -FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} - -COPY --from=upstream /usr/local/bin/* /usr/local/bin/ -COPY --from=upstream /var/lib/vector /var/lib/vector -COPY --from=upstream /etc/vector/vector.yaml /etc/vector/vector.yaml - -HEALTHCHECK NONE - -ENTRYPOINT ["/usr/local/bin/vector"] diff --git a/src/vector/README.md b/src/vector/README.md index 7c424cb33..d09241ca4 100644 --- a/src/vector/README.md +++ b/src/vector/README.md @@ -15,23 +15,23 @@ One of the main issues that has arisen with Promtail is its limited output/expor ### Goals and Options In choosing an alternative to Promtail we have a few primary objectives: -1. Chosen tool must be capable of gathering host and pod logs: This has been our primary usage of Promtail in the past - gathering pods logs and host logs (to include k8s audit logs, controlplane logs, etc). -1. Provide a tool that has numerous export options to cover specific needs for environments: Current known requirements include Loki, S3, and SIEM tools like Elastic and Splunk. Ideally the tool of choice supports all of these and more, allowing for expansion as new environments require it. -1. Choose a tool that does not require major changes in our logging stack, but is flexible for future adjustments to the stack: As we do have active users of our product we want to be careful in switching tools, so ideally we would like a tool that is a "drop-in" replacement. However, we don't want to rule out future changes to other pieces of the stack (i.e. Loki) so choosing a tool that doesn't lock us into Loki is important. -1. Focus on the log collection/shipping problem: While there are a number of tools that offer far more than just logging pipelines (metrics, traces, etc), we don't currently see a need to focus on these tools. These features are seen as a nice to have, but not being evaluated as the focus here. +- Chosen tool must be capable of gathering host and pod logs: This has been our primary usage of Promtail in the past - gathering pods logs and host logs (to include k8s audit logs, controlplane logs, etc). +- Provide a tool that has numerous export options to cover specific needs for environments: Current known requirements include Loki, S3, and SIEM tools like Elastic and Splunk. Ideally the tool of choice supports all of these and more, allowing for expansion as new environments require it. +- Choose a tool that does not require major changes in our logging stack, but is flexible for future adjustments to the stack: As we do have active users of our product we want to be careful in switching tools, so ideally we would like a tool that is a "drop-in" replacement. However, we don't want to rule out future changes to other pieces of the stack (i.e. Loki) so choosing a tool that doesn't lock us into Loki is important. +- Focus on the log collection/shipping problem: While there are a number of tools that offer far more than just logging pipelines (metrics, traces, etc), we don't currently see a need to focus on these tools. These features are seen as a nice to have, but not being evaluated as the focus here. Three tools in the space of log collection were considered: -1. [Vector](https://vector.dev/): Opensource and maintained by Datadog, Vector provides input integrations with Kubernetes logs, arbitrary files, and [other sources](https://vector.dev/docs/reference/configuration/sources/). It has the necessary export integrations with Loki, S3, Elastic, Splunk and a [number of other sinks](https://vector.dev/docs/reference/configuration/sinks/). Vector is a newer tool that has not yet reached a 1.0 release, but has risen in popularity due to its performance improvements over other tools. -1. [FluentBit](https://fluentbit.io/): Fluentbit was historically used in Big Bang and supports file based inputs as well as [other inputs](https://docs.fluentbit.io/manual/pipeline/inputs). It also supports the necessary output integrations (Loki, S3, Elastic, Splunk and [others](https://docs.fluentbit.io/manual/pipeline/outputs)). FluentBit is a CNCF graduated project and is relatively mature. Fluentbit fell out of favor with Big Bang due to some of the complexities around managing it at scale, specifically with its buffering. -1. [Grafana Alloy](https://grafana.com/docs/alloy/latest/): Alloy is a distribution of the OpenTelemetry Collector, opensource and maintained by Grafana Labs. It supports the necessary [inputs and outputs](https://grafana.com/docs/alloy/latest/reference/components/) (local file/k8s logs, Loki and S3). As a distribution of OTel it supports vendor-agnostic output formats and can be integrated with numerous other tools through the OTel ecosystem. While Alloy itself is relatively new, it is built on the previous codebase of Grafana Agent and the existing OTel framework. Notably it does not have any direct integrations with Splunk or Elastic, and its S3 integration is noted as experimental. +- [Vector](https://vector.dev/): Opensource and maintained by Datadog, Vector provides input integrations with Kubernetes logs, arbitrary files, and [other sources](https://vector.dev/docs/reference/configuration/sources/). It has the necessary export integrations with Loki, S3, Elastic, Splunk and a [number of other sinks](https://vector.dev/docs/reference/configuration/sinks/). Vector is a newer tool that has not yet reached a 1.0 release, but has risen in popularity due to its performance improvements over other tools. +- [FluentBit](https://fluentbit.io/): Fluentbit was historically used in Big Bang and supports file based inputs as well as [other inputs](https://docs.fluentbit.io/manual/pipeline/inputs). It also supports the necessary output integrations (Loki, S3, Elastic, Splunk and [others](https://docs.fluentbit.io/manual/pipeline/outputs)). FluentBit is a CNCF graduated project and is relatively mature. Fluentbit fell out of favor with Big Bang due to some of the complexities around managing it at scale, specifically with its buffering. +- [Grafana Alloy](https://grafana.com/docs/alloy/latest/): Alloy is a distribution of the OpenTelemetry Collector, opensource and maintained by Grafana Labs. It supports the necessary [inputs and outputs](https://grafana.com/docs/alloy/latest/reference/components/) (local file/k8s logs, Loki and S3). As a distribution of OTel it supports vendor-agnostic output formats and can be integrated with numerous other tools through the OTel ecosystem. While Alloy itself is relatively new, it is built on the previous codebase of Grafana Agent and the existing OTel framework. Notably it does not have any direct integrations with Splunk or Elastic, and its S3 integration is noted as experimental. ### Decision and Impact Vector has been chosen as our replacement for Promtail. Primary motivations include: -1. Vector has an extensive "component" catalog for inputs and outputs, with complete coverage of all currently desired export locations (and all are noted as "stable" integrations). -1. Vector's configuration is simple and works well in helm/with UDS helm overrides (easy to add additional export locations via bundle overrides for example). -1. Despite being a newer project, Vector's community is very active - with the most active contributors and GitHub stars compared to the other two tools. -1. Vector is [significantly more performant](https://github.com/vectordotdev/vector?tab=readme-ov-file#performance) than other tooling in the space on most categories of metrics. +- Vector has an extensive "component" catalog for inputs and outputs, with complete coverage of all currently desired export locations (and all are noted as "stable" integrations). +- Vector's configuration is simple and works well in helm/with UDS helm overrides (easy to add additional export locations via bundle overrides for example). +- Despite being a newer project, Vector's community is very active - with the most active contributors and GitHub stars compared to the other two tools. +- Vector is [significantly more performant](https://github.com/vectordotdev/vector?tab=readme-ov-file#performance) than other tooling in the space on most categories of metrics. As with any decisions of tooling in core this can always be reevaluated in the future as different tools or factors affect how we look at our logging stack.