From 3bdb4b9337729ee4690cef8ce7852d8152a17959 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 22 Aug 2024 11:36:10 -0600 Subject: [PATCH 01/22] wip: istio ambient --- src/istio/common/zarf.yaml | 12 +++++++- src/istio/values/base-cni.yaml | 3 ++ .../values/{values.yaml => base-istiod.yaml} | 0 src/istio/values/registry1/cni.yaml | 1 + .../istiod.yaml} | 0 src/istio/values/registry1/ztunnel.yaml | 1 + src/istio/values/unicorn/cni.yaml | 1 + .../istiod.yaml} | 0 src/istio/values/upstream/cni.yaml | 1 + .../istiod.yaml} | 0 src/istio/values/upstream/ztunnel.yaml | 1 + src/istio/zarf.yaml | 30 +++++++++++++++++-- 12 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 src/istio/values/base-cni.yaml rename src/istio/values/{values.yaml => base-istiod.yaml} (100%) create mode 100644 src/istio/values/registry1/cni.yaml rename src/istio/values/{registry1-values.yaml => registry1/istiod.yaml} (100%) create mode 100644 src/istio/values/registry1/ztunnel.yaml create mode 100644 src/istio/values/unicorn/cni.yaml rename src/istio/values/{unicorn-values.yaml => unicorn/istiod.yaml} (100%) create mode 100644 src/istio/values/upstream/cni.yaml rename src/istio/values/{upstream-values.yaml => upstream/istiod.yaml} (100%) create mode 100644 src/istio/values/upstream/ztunnel.yaml diff --git a/src/istio/common/zarf.yaml b/src/istio/common/zarf.yaml index 55417693b..2cf247742 100644 --- a/src/istio/common/zarf.yaml +++ b/src/istio/common/zarf.yaml @@ -17,7 +17,17 @@ components: version: 1.22.3 namespace: istio-system valuesFiles: - - "../values/values.yaml" + - "../values/base-istiod.yaml" + - name: cni + url: https://istio-release.storage.googleapis.com/charts + version: 1.22.3 + namespace: istio-system + valuesFiles: + - "../values/base-cni.yaml" # values for k3s/k3d cni + - name: ztunnel + url: https://istio-release.storage.googleapis.com/charts + version: 1.22.3 + namespace: istio-system manifests: - name: uds-global-istio-config namespace: istio-system diff --git a/src/istio/values/base-cni.yaml b/src/istio/values/base-cni.yaml new file mode 100644 index 000000000..17f81b334 --- /dev/null +++ b/src/istio/values/base-cni.yaml @@ -0,0 +1,3 @@ +cni: + cniConfDir: /var/lib/rancher/k3s/agent/etc/cni/net.d + cniBinDir: /bin/ diff --git a/src/istio/values/values.yaml b/src/istio/values/base-istiod.yaml similarity index 100% rename from src/istio/values/values.yaml rename to src/istio/values/base-istiod.yaml diff --git a/src/istio/values/registry1/cni.yaml b/src/istio/values/registry1/cni.yaml new file mode 100644 index 000000000..cef8f7d38 --- /dev/null +++ b/src/istio/values/registry1/cni.yaml @@ -0,0 +1 @@ +image: registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.22.3 diff --git a/src/istio/values/registry1-values.yaml b/src/istio/values/registry1/istiod.yaml similarity index 100% rename from src/istio/values/registry1-values.yaml rename to src/istio/values/registry1/istiod.yaml diff --git a/src/istio/values/registry1/ztunnel.yaml b/src/istio/values/registry1/ztunnel.yaml new file mode 100644 index 000000000..2ed685368 --- /dev/null +++ b/src/istio/values/registry1/ztunnel.yaml @@ -0,0 +1 @@ +image: registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.22.3 diff --git a/src/istio/values/unicorn/cni.yaml b/src/istio/values/unicorn/cni.yaml new file mode 100644 index 000000000..69fe48df9 --- /dev/null +++ b/src/istio/values/unicorn/cni.yaml @@ -0,0 +1 @@ +image: cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.22.3 diff --git a/src/istio/values/unicorn-values.yaml b/src/istio/values/unicorn/istiod.yaml similarity index 100% rename from src/istio/values/unicorn-values.yaml rename to src/istio/values/unicorn/istiod.yaml diff --git a/src/istio/values/upstream/cni.yaml b/src/istio/values/upstream/cni.yaml new file mode 100644 index 000000000..8b38b7e12 --- /dev/null +++ b/src/istio/values/upstream/cni.yaml @@ -0,0 +1 @@ +image: docker.io/istio/install-cni:1.22.3-distroless diff --git a/src/istio/values/upstream-values.yaml b/src/istio/values/upstream/istiod.yaml similarity index 100% rename from src/istio/values/upstream-values.yaml rename to src/istio/values/upstream/istiod.yaml diff --git a/src/istio/values/upstream/ztunnel.yaml b/src/istio/values/upstream/ztunnel.yaml new file mode 100644 index 000000000..15751d1f5 --- /dev/null +++ b/src/istio/values/upstream/ztunnel.yaml @@ -0,0 +1 @@ +image: docker.io/istio/ztunnel:1.22.3-distroless diff --git a/src/istio/zarf.yaml b/src/istio/zarf.yaml index 9ae317783..e37088143 100644 --- a/src/istio/zarf.yaml +++ b/src/istio/zarf.yaml @@ -19,10 +19,18 @@ components: charts: - name: istiod valuesFiles: - - "values/upstream-values.yaml" + - "values/upstream/istiod.yaml" + - name: cni + valuesFiles: + - "values/upstream/cni.yaml" + - name: ztunnel + valuesFiles: + - "values/upstream/ztunnel.yaml" images: - "docker.io/istio/pilot:1.22.3-distroless" - "docker.io/istio/proxyv2:1.22.3-distroless" + - "docker.io/istio/install-cni:1.22.3-distroless" + - "docker.io/istio/ztunnel:1.22.3-distroless" - name: istio-controlplane required: true @@ -33,10 +41,18 @@ components: charts: - name: istiod valuesFiles: - - "values/registry1-values.yaml" + - "values/registry1/istiod.yaml" + - name: cni + valuesFiles: + - "values/registry1/cni.yaml" + - name: ztunnel + valuesFiles: + - "values/registry1/ztunnel.yaml" images: - registry1.dso.mil/ironbank/tetrate/istio/proxyv2:1.22.3-tetratefips-v0 - registry1.dso.mil/ironbank/tetrate/istio/pilot:1.22.3-tetratefips-v0 + - registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.22.3 + - registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.22.3 - name: istio-controlplane required: true @@ -47,10 +63,18 @@ components: charts: - name: istiod valuesFiles: - - "values/unicorn-values.yaml" + - "values/unicorn/istiod.yaml" + - name: cni + valuesFiles: + - "values/unicorn/cni.yaml" + - name: ztunnel + valuesFiles: + - "values/upstream/ztunnel.yaml" images: - cgr.dev/du-uds-defenseunicorns/istio-pilot-fips:1.22.3 - cgr.dev/du-uds-defenseunicorns/istio-proxy-fips:1.22.3 + - cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.22.3 + - docker.io/istio/ztunnel:1.22.3-distroless - name: istio-admin-gateway required: true From 432d31b8b1001377779c92e40d371a995326035a Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 22 Aug 2024 11:46:52 -0600 Subject: [PATCH 02/22] fix: cni images --- src/istio/values/registry1/cni.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/istio/values/registry1/cni.yaml b/src/istio/values/registry1/cni.yaml index cef8f7d38..10ad6cb3f 100644 --- a/src/istio/values/registry1/cni.yaml +++ b/src/istio/values/registry1/cni.yaml @@ -1 +1,2 @@ -image: registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.22.3 +cni: + image: registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.22.3 From a647971175a44b63953ad3004cf09d2118c334aa Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 22 Aug 2024 11:47:27 -0600 Subject: [PATCH 03/22] fix: cni images --- src/istio/values/unicorn/cni.yaml | 3 ++- src/istio/values/upstream/cni.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/istio/values/unicorn/cni.yaml b/src/istio/values/unicorn/cni.yaml index 69fe48df9..eb819c81b 100644 --- a/src/istio/values/unicorn/cni.yaml +++ b/src/istio/values/unicorn/cni.yaml @@ -1 +1,2 @@ -image: cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.22.3 +cni: + image: cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.22.3 diff --git a/src/istio/values/upstream/cni.yaml b/src/istio/values/upstream/cni.yaml index 8b38b7e12..9ef21c6c5 100644 --- a/src/istio/values/upstream/cni.yaml +++ b/src/istio/values/upstream/cni.yaml @@ -1 +1,2 @@ -image: docker.io/istio/install-cni:1.22.3-distroless +cni: + image: docker.io/istio/install-cni:1.22.3-distroless From b363ab2a4ad37fcce0ddaba0bfae579f224ecd1f Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 22 Aug 2024 11:58:25 -0600 Subject: [PATCH 04/22] fix: profile ambient --- src/istio/values/base-cni.yaml | 2 ++ src/istio/values/base-istiod.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/istio/values/base-cni.yaml b/src/istio/values/base-cni.yaml index 17f81b334..1e5b6af8e 100644 --- a/src/istio/values/base-cni.yaml +++ b/src/istio/values/base-cni.yaml @@ -1,3 +1,5 @@ +profile: ambient + cni: cniConfDir: /var/lib/rancher/k3s/agent/etc/cni/net.d cniBinDir: /bin/ diff --git a/src/istio/values/base-istiod.yaml b/src/istio/values/base-istiod.yaml index 4b5412489..451ae3d9e 100644 --- a/src/istio/values/base-istiod.yaml +++ b/src/istio/values/base-istiod.yaml @@ -1,3 +1,5 @@ +profile: ambient + meshConfig: accessLogFile: /dev/stdout pathNormalization: From 849db8d2e606bc4f8596988c373256961038bb40 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 22 Aug 2024 13:56:03 -0600 Subject: [PATCH 05/22] wip: more ztunnels --- src/istio/zarf.yaml | 12 ++++++++---- src/pepr/operator/controllers/istio/injection.ts | 5 ++++- .../network/defaults/allow-ingress-ztunnel.ts | 10 ++++++++++ src/pepr/operator/controllers/network/policies.ts | 2 ++ src/test/app-admin.yaml | 2 ++ src/test/app-authservice-tenant.yaml | 2 ++ src/test/app-tenant.yaml | 2 ++ 7 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 src/pepr/operator/controllers/network/defaults/allow-ingress-ztunnel.ts diff --git a/src/istio/zarf.yaml b/src/istio/zarf.yaml index e37088143..563f4e075 100644 --- a/src/istio/zarf.yaml +++ b/src/istio/zarf.yaml @@ -47,11 +47,13 @@ components: - "values/registry1/cni.yaml" - name: ztunnel valuesFiles: - - "values/registry1/ztunnel.yaml" + # - "values/registry1/ztunnel.yaml" + - "values/upstream/ztunnel.yaml" images: - registry1.dso.mil/ironbank/tetrate/istio/proxyv2:1.22.3-tetratefips-v0 - registry1.dso.mil/ironbank/tetrate/istio/pilot:1.22.3-tetratefips-v0 - - registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.22.3 + # - registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.22.3 + - docker.io/istio/ztunnel:1.22.3-distroless - registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.22.3 - name: istio-controlplane @@ -66,14 +68,16 @@ components: - "values/unicorn/istiod.yaml" - name: cni valuesFiles: - - "values/unicorn/cni.yaml" + # - "values/unicorn/cni.yaml" + - "values/upstream/cni.yaml" - name: ztunnel valuesFiles: - "values/upstream/ztunnel.yaml" images: - cgr.dev/du-uds-defenseunicorns/istio-pilot-fips:1.22.3 - cgr.dev/du-uds-defenseunicorns/istio-proxy-fips:1.22.3 - - cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.22.3 + - docker.io/istio/install-cni:1.22.3-distroless + # - cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.22.3 - docker.io/istio/ztunnel:1.22.3-distroless - name: istio-admin-gateway diff --git a/src/pepr/operator/controllers/istio/injection.ts b/src/pepr/operator/controllers/istio/injection.ts index 50193ea96..02f9f57cb 100644 --- a/src/pepr/operator/controllers/istio/injection.ts +++ b/src/pepr/operator/controllers/istio/injection.ts @@ -31,7 +31,10 @@ export async function enableInjection(pkg: UDSPackage) { } // Ensure the namespace is configured - if (!annotations[pkgKey] || originalInjectionLabel !== "enabled") { + if ( + (!annotations[pkgKey] || originalInjectionLabel !== "enabled") && + labels["istio.io/dataplane-mode"] !== "ambient" + ) { // Ensure Istio injection is enabled labels[injectionLabel] = "enabled"; diff --git a/src/pepr/operator/controllers/network/defaults/allow-ingress-ztunnel.ts b/src/pepr/operator/controllers/network/defaults/allow-ingress-ztunnel.ts new file mode 100644 index 000000000..2cd069065 --- /dev/null +++ b/src/pepr/operator/controllers/network/defaults/allow-ingress-ztunnel.ts @@ -0,0 +1,10 @@ +import { Direction } from "../../../crd"; +import { generate } from "../generate"; + +export const allowIngressZtunnel = (namespace: string) => + generate(namespace, { + direction: Direction.Ingress, + description: "Ztunnel", + remoteNamespace: "*", + port: 15008, + }); diff --git a/src/pepr/operator/controllers/network/policies.ts b/src/pepr/operator/controllers/network/policies.ts index 8534f028f..e9d7e569f 100644 --- a/src/pepr/operator/controllers/network/policies.ts +++ b/src/pepr/operator/controllers/network/policies.ts @@ -6,6 +6,7 @@ import { getOwnerRef, purgeOrphans, sanitizeResourceName } from "../utils"; import { allowEgressDNS } from "./defaults/allow-egress-dns"; import { allowEgressIstiod } from "./defaults/allow-egress-istiod"; import { allowIngressSidecarMonitoring } from "./defaults/allow-ingress-sidecar-monitoring"; +import { allowIngressZtunnel } from "./defaults/allow-ingress-ztunnel"; import { defaultDenyAll } from "./defaults/default-deny-all"; import { generate } from "./generate"; @@ -31,6 +32,7 @@ export async function networkPolicies(pkg: UDSPackage, namespace: string) { // Istio rules allowEgressIstiod(namespace), + allowIngressZtunnel(namespace), allowIngressSidecarMonitoring(namespace), ]; diff --git a/src/test/app-admin.yaml b/src/test/app-admin.yaml index 884401949..51073dc6b 100644 --- a/src/test/app-admin.yaml +++ b/src/test/app-admin.yaml @@ -2,6 +2,8 @@ apiVersion: v1 kind: Namespace metadata: name: test-admin-app + labels: + istio.io/dataplane-mode: ambient --- apiVersion: uds.dev/v1alpha1 kind: Package diff --git a/src/test/app-authservice-tenant.yaml b/src/test/app-authservice-tenant.yaml index 094bff22e..f05df4787 100644 --- a/src/test/app-authservice-tenant.yaml +++ b/src/test/app-authservice-tenant.yaml @@ -2,6 +2,8 @@ apiVersion: v1 kind: Namespace metadata: name: authservice-test-app + labels: + istio.io/dataplane-mode: ambient --- apiVersion: uds.dev/v1alpha1 kind: Package diff --git a/src/test/app-tenant.yaml b/src/test/app-tenant.yaml index 3eb203b99..3a51b7b40 100644 --- a/src/test/app-tenant.yaml +++ b/src/test/app-tenant.yaml @@ -2,6 +2,8 @@ apiVersion: v1 kind: Namespace metadata: name: test-tenant-app + labels: + istio.io/dataplane-mode: ambient --- apiVersion: uds.dev/v1alpha1 kind: Package From a3503c81fad0859e4a895580305b4b4835d6d55b Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 22 Aug 2024 14:06:37 -0600 Subject: [PATCH 06/22] wip: reg1 image --- src/istio/zarf.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/istio/zarf.yaml b/src/istio/zarf.yaml index 563f4e075..f12c3d4a3 100644 --- a/src/istio/zarf.yaml +++ b/src/istio/zarf.yaml @@ -44,7 +44,8 @@ components: - "values/registry1/istiod.yaml" - name: cni valuesFiles: - - "values/registry1/cni.yaml" + # - "values/registry1/cni.yaml" + - "values/upstream/cni.yaml" - name: ztunnel valuesFiles: # - "values/registry1/ztunnel.yaml" @@ -54,7 +55,8 @@ components: - registry1.dso.mil/ironbank/tetrate/istio/pilot:1.22.3-tetratefips-v0 # - registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.22.3 - docker.io/istio/ztunnel:1.22.3-distroless - - registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.22.3 + # - registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.22.3 + - docker.io/istio/install-cni:1.22.3-distroless - name: istio-controlplane required: true From c3859b20bce756994ce739f03fb098f3c9864e0b Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 22 Aug 2024 15:15:40 -0600 Subject: [PATCH 07/22] wip: injection -> ambient --- .../operator/controllers/istio/injection.ts | 30 +++++++++++-------- src/test/app-authservice-tenant.yaml | 4 +-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/pepr/operator/controllers/istio/injection.ts b/src/pepr/operator/controllers/istio/injection.ts index 02f9f57cb..1fcc78249 100644 --- a/src/pepr/operator/controllers/istio/injection.ts +++ b/src/pepr/operator/controllers/istio/injection.ts @@ -7,6 +7,7 @@ import { UDSPackage } from "../../crd"; const log = setupLogger(Component.OPERATOR_ISTIO); const injectionLabel = "istio-injection"; +const ambientLabel = "istio.io/dataplane-mode"; const injectionAnnotation = "uds.dev/original-istio-injection"; /** @@ -21,6 +22,7 @@ export async function enableInjection(pkg: UDSPackage) { const sourceNS = await K8s(kind.Namespace).Get(pkg.metadata.namespace); const labels = sourceNS.metadata?.labels || {}; + const originalAmbientLabel = labels[ambientLabel]; const originalInjectionLabel = labels[injectionLabel]; const annotations = sourceNS.metadata?.annotations || {}; const pkgKey = `uds.dev/pkg-${pkg.metadata.name}`; @@ -31,18 +33,16 @@ export async function enableInjection(pkg: UDSPackage) { } // Ensure the namespace is configured - if ( - (!annotations[pkgKey] || originalInjectionLabel !== "enabled") && - labels["istio.io/dataplane-mode"] !== "ambient" - ) { - // Ensure Istio injection is enabled - labels[injectionLabel] = "enabled"; + if (!annotations[pkgKey] || originalAmbientLabel !== "ambient") { + // Ensure Istio ambient is enabled + labels[ambientLabel] = "ambient"; + delete labels[injectionLabel]; // Add the package annotation annotations[pkgKey] = "true"; // Apply the updated Namespace - log.debug(`Updating namespace ${pkg.metadata.namespace} with istio injection label`); + log.debug(`Updating namespace ${pkg.metadata.namespace} with istio ambient label`); await K8s(kind.Namespace).Apply( { metadata: { @@ -54,12 +54,12 @@ export async function enableInjection(pkg: UDSPackage) { { force: true }, ); - // Kill the pods if we changed the value of the istio-injection label - if (originalInjectionLabel !== labels[injectionLabel]) { + // Kill the pods if we are switching from injection to ambient + if (originalInjectionLabel === "enabled") { log.debug( `Attempting pod restart in ${pkg.metadata.namespace} based on istio injection label change`, ); - await killPods(pkg.metadata.namespace, true); + await killPods(pkg.metadata.namespace, false); } } } @@ -93,7 +93,9 @@ export async function cleanupNamespace(pkg: UDSPackage) { } // Apply the updated Namespace - log.debug(`Updating namespace ${pkg.metadata.namespace}, removing istio injection labels.`); + log.debug( + `Updating namespace ${pkg.metadata.namespace}, applying original istio injection labels.`, + ); await K8s(kind.Namespace).Apply( { metadata: { @@ -110,7 +112,7 @@ export async function cleanupNamespace(pkg: UDSPackage) { log.debug( `Attempting pod restart in ${pkg.metadata.namespace} based on istio injection label change`, ); - await killPods(pkg.metadata.namespace, false); + await killPods(pkg.metadata.namespace, true); } } @@ -162,7 +164,9 @@ async function killPods(ns: string, enableInjection: boolean) { } for (const pod of group) { - log.info(`Deleting pod ${ns}/${pod.metadata?.name} to enable the istio sidecar`); + log.info( + `Deleting pod ${ns}/${pod.metadata?.name} to ${enableInjection ? "enable" : "disable"} the istio sidecar`, + ); await K8s(kind.Pod).Delete(pod); } } diff --git a/src/test/app-authservice-tenant.yaml b/src/test/app-authservice-tenant.yaml index f05df4787..53bb92198 100644 --- a/src/test/app-authservice-tenant.yaml +++ b/src/test/app-authservice-tenant.yaml @@ -2,8 +2,8 @@ apiVersion: v1 kind: Namespace metadata: name: authservice-test-app - labels: - istio.io/dataplane-mode: ambient + # labels: + # istio.io/dataplane-mode: ambient --- apiVersion: uds.dev/v1alpha1 kind: Package From cfcf1c1b9a0f409772590be998c5d0496a66dc30 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Fri, 23 Aug 2024 09:43:16 -0600 Subject: [PATCH 08/22] wip: delete delete delete --- pepr.ts | 4 - .../common/manifests/pepr-istio-config.yaml | 35 +---- src/istio/common/zarf.yaml | 7 +- src/istio/tasks.yaml | 6 +- src/keycloak/chart/templates/istio-admin.yaml | 134 +++++++++--------- src/keycloak/chart/values.yaml | 2 +- src/keycloak/zarf.yaml | 6 +- .../loki-simple-scalable.yaml | 17 --- .../peerauthentication/metrics-api.yaml | 17 --- .../istio/headless-controller-service.yaml | 28 ---- .../istio/headless-enforcer-service.yaml | 21 --- .../istio/headless-scanner-service.yaml | 15 -- .../neuvector-controller-pa.yaml | 19 --- src/pepr/prometheus/index.ts | 108 -------------- .../chart/templates/istio-monitor.yaml | 2 - .../prometheus-operator-pa.yaml | 17 --- .../templates/prometheus-pod-monitor.yaml | 2 - src/prometheus-stack/values/values.yaml | 25 ---- 18 files changed, 84 insertions(+), 381 deletions(-) delete mode 100644 src/loki/chart/templates/peerauthentication/loki-simple-scalable.yaml delete mode 100644 src/metrics-server/chart/templates/peerauthentication/metrics-api.yaml delete mode 100644 src/neuvector/chart/templates/istio/headless-controller-service.yaml delete mode 100644 src/neuvector/chart/templates/istio/headless-enforcer-service.yaml delete mode 100644 src/neuvector/chart/templates/istio/headless-scanner-service.yaml delete mode 100644 src/neuvector/chart/templates/peerauthentication/neuvector-controller-pa.yaml delete mode 100644 src/pepr/prometheus/index.ts delete mode 100644 src/prometheus-stack/chart/templates/peerauthentication/prometheus-operator-pa.yaml diff --git a/pepr.ts b/pepr.ts index 885b09d3d..bb682d636 100644 --- a/pepr.ts +++ b/pepr.ts @@ -10,7 +10,6 @@ import { setupAuthserviceSecret } from "./src/pepr/operator/controllers/keycloak import { Policy } from "./src/pepr/operator/crd"; import { registerCRDs } from "./src/pepr/operator/crd/register"; import { policies, startExemptionWatch } from "./src/pepr/policies"; -import { prometheus } from "./src/pepr/prometheus"; const log = setupLogger(Component.STARTUP); @@ -29,9 +28,6 @@ const log = setupLogger(Component.STARTUP); // Istio service mesh istio, - - // Prometheus monitoring stack - prometheus, ]); // Remove legacy policy entries from the pepr store for the 0.5.0 upgrade if ( diff --git a/src/istio/common/manifests/pepr-istio-config.yaml b/src/istio/common/manifests/pepr-istio-config.yaml index 50eaf2ee1..683fd3dc5 100644 --- a/src/istio/common/manifests/pepr-istio-config.yaml +++ b/src/istio/common/manifests/pepr-istio-config.yaml @@ -1,38 +1,9 @@ -# Have to pre-create the namespace and also patch it with the istio-injection label later because +# Have to pre-create the namespace and also patch it with the istio ambient label later because # Helm is kind of dumb: https://github.com/helm/helm/issues/350 kind: Namespace apiVersion: v1 metadata: name: pepr-system labels: - istio-injection: enabled ---- -apiVersion: "security.istio.io/v1beta1" -kind: PeerAuthentication -metadata: - name: permissive-pepr-webhook - namespace: pepr-system -spec: - selector: - matchLabels: - pepr.dev/controller: admission - mtls: - mode: STRICT - portLevelMtls: - "3000": - mode: PERMISSIVE ---- -apiVersion: "security.istio.io/v1beta1" -kind: PeerAuthentication -metadata: - name: permissive-pepr-webhook-watcher - namespace: pepr-system -spec: - selector: - matchLabels: - pepr.dev/controller: watcher - mtls: - mode: STRICT - portLevelMtls: - "3000": - mode: PERMISSIVE + istio-injection: disabled + istio.io/dataplane-mode: ambient diff --git a/src/istio/common/zarf.yaml b/src/istio/common/zarf.yaml index 2cf247742..043f732fe 100644 --- a/src/istio/common/zarf.yaml +++ b/src/istio/common/zarf.yaml @@ -35,8 +35,11 @@ components: - "manifests/envoy-filters.yaml" - "manifests/peer-authentication.yaml" - "manifests/pepr-istio-config.yaml" + - https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml actions: onDeploy: after: - - description: "Ensure istio-injection is enabled for Pepr" - cmd: "./zarf tools kubectl label namespace pepr-system istio-injection=enabled --overwrite" + - description: "Ensure istio ambient is enabled for Pepr" + cmd: "./zarf tools kubectl label namespace pepr-system istio.io/dataplane-mode=ambient --overwrite" + - description: "Ensure istio-injection is disabled for Pepr" + cmd: "./zarf tools kubectl label namespace pepr-system istio-injection=disabled --overwrite" diff --git a/src/istio/tasks.yaml b/src/istio/tasks.yaml index a41c5ec03..ec9d28517 100644 --- a/src/istio/tasks.yaml +++ b/src/istio/tasks.yaml @@ -4,20 +4,20 @@ tasks: - description: Validate the Istio Admin Gateway wait: cluster: - kind: Gateway + kind: gateways.networking.istio.io name: admin-gateway namespace: istio-admin-gateway - description: Validate the Istio Passthrough Gateway wait: cluster: - kind: Gateway + kind: gateways.networking.istio.io name: passthrough-gateway namespace: istio-passthrough-gateway - description: Validate the Istio Tenant Gateway wait: cluster: - kind: Gateway + kind: gateways.networking.istio.io name: tenant-gateway namespace: istio-tenant-gateway diff --git a/src/keycloak/chart/templates/istio-admin.yaml b/src/keycloak/chart/templates/istio-admin.yaml index 54f6ff114..09e9319f3 100644 --- a/src/keycloak/chart/templates/istio-admin.yaml +++ b/src/keycloak/chart/templates/istio-admin.yaml @@ -1,65 +1,69 @@ -{{- if .Capabilities.APIVersions.Has "security.istio.io/v1beta1" }} -apiVersion: security.istio.io/v1beta1 -kind: AuthorizationPolicy -metadata: - name: keycloak-block-admin-access-from-public-gateway - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - {{- include "keycloak.selectorLabels" . | nindent 6 }} - action: DENY - rules: - - to: - - operation: - ports: - - "8080" - paths: - - "/admin*" - - "/realms/master*" - from: - - source: - notNamespaces: - - istio-admin-gateway - - to: - - operation: - ports: - - "8080" - paths: - - /metrics* - from: - - source: - notNamespaces: - - istio-admin-gateway - - monitoring - - to: - - operation: - ports: - - "8080" - paths: - # Never allow anonymous client registration except from the pepr-system namespace - # This is another fallback protection, as the KC policy already blocks it - - "/realms/{{ .Values.realm }}/clients-registrations/*" - from: - - source: - notNamespaces: - - "pepr-system" - - when: - - key: request.headers[istio-mtls-client-certificate] - values: ["*"] - to: - - operation: - ports: - - "8080" - from: - - source: - notNamespaces: - - istio-tenant-gateway - - istio-admin-gateway - {{- range .Values.additionalGatewayNamespaces }} - {{- if not (hasPrefix "istio-" .) }} - {{- fail (printf "Allowed gateway namespace '%s' must start with 'istio-'" .) }} - {{- end }} - - {{ . }} - {{- end }} -{{- end }} +# {{- if .Capabilities.APIVersions.Has "security.istio.io/v1beta1" }} +# apiVersion: security.istio.io/v1beta1 +# kind: AuthorizationPolicy +# metadata: +# name: keycloak-block-admin-access-from-public-gateway +# namespace: {{ .Release.Namespace }} +# spec: +# targetRefs: +# - kind: Service +# group: "" +# name: keycloak-http +# - kind: Service +# group: "" +# name: keycloak-headless +# action: DENY +# rules: +# - to: +# - operation: +# ports: +# - "8080" +# paths: +# - "/admin*" +# - "/realms/master*" +# from: +# - source: +# notNamespaces: +# - istio-admin-gateway +# - to: +# - operation: +# ports: +# - "8080" +# paths: +# - /metrics* +# from: +# - source: +# notNamespaces: +# - istio-admin-gateway +# - monitoring +# - to: +# - operation: +# ports: +# - "8080" +# paths: +# # Never allow anonymous client registration except from the pepr-system namespace +# # This is another fallback protection, as the KC policy already blocks it +# - "/realms/{{ .Values.realm }}/clients-registrations/*" +# from: +# - source: +# notNamespaces: +# - "pepr-system" +# - when: +# - key: request.headers[istio-mtls-client-certificate] +# values: ["*"] +# to: +# - operation: +# ports: +# - "8080" +# from: +# - source: +# notNamespaces: +# - istio-tenant-gateway +# - istio-admin-gateway +# {{- range .Values.additionalGatewayNamespaces }} +# {{- if not (hasPrefix "istio-" .) }} +# {{- fail (printf "Allowed gateway namespace '%s' must start with 'istio-'" .) }} +# {{- end }} +# - {{ . }} +# {{- end }} +# {{- end }} diff --git a/src/keycloak/chart/values.yaml b/src/keycloak/chart/values.yaml index 08924801f..949483e69 100644 --- a/src/keycloak/chart/values.yaml +++ b/src/keycloak/chart/values.yaml @@ -7,7 +7,7 @@ image: pullPolicy: IfNotPresent # renovate: datasource=github-tags depName=defenseunicorns/uds-identity-config versioning=semver -configImage: ghcr.io/defenseunicorns/uds/identity-config:0.6.0 +configImage: rjferguson21/uds-core-config:ambient # The public domain name of the Keycloak server domain: "###ZARF_VAR_DOMAIN###" diff --git a/src/keycloak/zarf.yaml b/src/keycloak/zarf.yaml index 860137046..495622642 100644 --- a/src/keycloak/zarf.yaml +++ b/src/keycloak/zarf.yaml @@ -21,7 +21,7 @@ components: - "values/upstream-values.yaml" images: - quay.io/keycloak/keycloak:24.0.5 - - ghcr.io/defenseunicorns/uds/identity-config:0.6.0 + - rjferguson21/uds-core-config:ambient - name: keycloak required: true @@ -37,7 +37,7 @@ components: - "values/registry1-values.yaml" images: - registry1.dso.mil/ironbank/opensource/keycloak/keycloak:24.0.5 - - ghcr.io/defenseunicorns/uds/identity-config:0.6.0 + - rjferguson21/uds-core-config:ambient - name: keycloak required: true @@ -51,4 +51,4 @@ components: - "values/unicorn-values.yaml" images: - cgr.dev/du-uds-defenseunicorns/keycloak:24.0.5 # todo: switch to FIPS image - - ghcr.io/defenseunicorns/uds/identity-config:0.6.0 + - rjferguson21/uds-core-config:ambient diff --git a/src/loki/chart/templates/peerauthentication/loki-simple-scalable.yaml b/src/loki/chart/templates/peerauthentication/loki-simple-scalable.yaml deleted file mode 100644 index 90f748d95..000000000 --- a/src/loki/chart/templates/peerauthentication/loki-simple-scalable.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Capabilities.APIVersions.Has "security.istio.io/v1beta1" }} -apiVersion: "security.istio.io/v1beta1" -kind: PeerAuthentication -metadata: - name: loki-simple-scalable - namespace: {{ .Release.Namespace }} -spec: - mtls: - mode: STRICT - selector: - matchLabels: - app.kubernetes.io/name: loki - portLevelMtls: - # GRPC exception to support Loki internal communication - "9095": - mode: PERMISSIVE -{{- end }} diff --git a/src/metrics-server/chart/templates/peerauthentication/metrics-api.yaml b/src/metrics-server/chart/templates/peerauthentication/metrics-api.yaml deleted file mode 100644 index 18c5fdcbf..000000000 --- a/src/metrics-server/chart/templates/peerauthentication/metrics-api.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Capabilities.APIVersions.Has "security.istio.io/v1beta1" }} -apiVersion: security.istio.io/v1beta1 -kind: PeerAuthentication -metadata: - name: metrics-server-api-exception - namespace: {{ .Release.Namespace }} -spec: - mtls: - mode: STRICT - selector: - matchLabels: - app.kubernetes.io/name: metrics-server - portLevelMtls: - # Allow api service calls to operate permissive since ingress originates from the nodes - "10250": - mode: PERMISSIVE -{{- end }} diff --git a/src/neuvector/chart/templates/istio/headless-controller-service.yaml b/src/neuvector/chart/templates/istio/headless-controller-service.yaml deleted file mode 100644 index 4e41a72d7..000000000 --- a/src/neuvector/chart/templates/istio/headless-controller-service.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: controller-headless - namespace: {{ .Release.Namespace }} -spec: - type: ClusterIP - clusterIP: None - selector: - app: neuvector-controller-pod - ports: - - name: tcp-gossip-18300 - port: 18300 - protocol: TCP - targetPort: 18300 - - name: tcp-gossip-18301 - port: 18301 - protocol: TCP - targetPort: 18301 - - name: tcp-controller-grpc - port: 18400 - protocol: TCP - targetPort: 18400 - - name: tcp-controller-api - port: 10443 - protocol: TCP - targetPort: 10443 - diff --git a/src/neuvector/chart/templates/istio/headless-enforcer-service.yaml b/src/neuvector/chart/templates/istio/headless-enforcer-service.yaml deleted file mode 100644 index 66c3f4513..000000000 --- a/src/neuvector/chart/templates/istio/headless-enforcer-service.yaml +++ /dev/null @@ -1,21 +0,0 @@ - -apiVersion: v1 -kind: Service -metadata: - name: enforcer-headless - namespace: {{ .Release.Namespace }} -spec: - type: ClusterIP - clusterIP: None - selector: - app: neuvector-enforcer-pod - ports: - - name: tcp-gossip - port: 18301 - protocol: TCP - targetPort: 18301 - - name: tcp-enforcer - port: 18401 - protocol: TCP - targetPort: 18401 - diff --git a/src/neuvector/chart/templates/istio/headless-scanner-service.yaml b/src/neuvector/chart/templates/istio/headless-scanner-service.yaml deleted file mode 100644 index f0d9da3b8..000000000 --- a/src/neuvector/chart/templates/istio/headless-scanner-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: scanner-headless - namespace: {{ .Release.Namespace }} -spec: - type: ClusterIP - clusterIP: None - selector: - app: neuvector-scanner-pod - ports: - - name: tcp-scanner - port: 18402 - protocol: TCP - targetPort: 18402 diff --git a/src/neuvector/chart/templates/peerauthentication/neuvector-controller-pa.yaml b/src/neuvector/chart/templates/peerauthentication/neuvector-controller-pa.yaml deleted file mode 100644 index 6e881bf73..000000000 --- a/src/neuvector/chart/templates/peerauthentication/neuvector-controller-pa.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if .Capabilities.APIVersions.Has "security.istio.io/v1beta1" }} -apiVersion: "security.istio.io/v1beta1" -kind: PeerAuthentication -metadata: - name: controller-neuvector - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - app: neuvector-controller-pod - mtls: - mode: STRICT - portLevelMtls: - "18300": - mode: PERMISSIVE - # Allow webhooks to operate permissive since ingress originates from the nodes - "30443": - mode: PERMISSIVE -{{- end }} diff --git a/src/pepr/prometheus/index.ts b/src/pepr/prometheus/index.ts deleted file mode 100644 index ef4c1e34e..000000000 --- a/src/pepr/prometheus/index.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { Capability, K8s, kind } from "pepr"; -import { Component, setupLogger } from "../logger"; -import { - PodMonitorEndpoint, - PodMonitorScheme, - PrometheusPodMonitor, - PrometheusServiceMonitor, - ServiceMonitorEndpoint, - ServiceMonitorScheme, -} from "../operator/crd"; -// configure subproject logger -const log = setupLogger(Component.PROMETHEUS); - -export const prometheus = new Capability({ - name: "prometheus", - description: "UDS Core Capability for the Prometheus stack.", -}); - -const { When } = prometheus; - -/** - * Mutate a service monitor to exclude it from mTLS metrics with `exempt` scrapeClass - */ -When(PrometheusServiceMonitor) - .IsCreatedOrUpdated() - .Mutate(async sm => { - if (sm.Raw.spec === undefined || sm.Raw.spec.scrapeClass != undefined) { - return; - } - - // Add an exempt scrape class if explicitly opted out via annotation OR targeting a non-istio-injected namespace - if ( - sm.Raw.metadata?.annotations?.["uds/skip-mutate"] || - sm.Raw.metadata?.annotations?.["uds/skip-sm-mutate"] || - !(await isIstioInjected(sm)) - ) { - log.info( - `Mutating scrapeClass to exempt ServiceMonitor ${sm.Raw.metadata?.name} from default scrapeClass mTLS config`, - ); - sm.Raw.spec.scrapeClass = "exempt"; - - return; - } else { - log.info(`Patching service monitor ${sm.Raw.metadata?.name} for mTLS metrics`); - // Note: this tlsConfig patch is deprecated in favor of a default scrape class for both service and pod monitors - const tlsConfig = { - caFile: "/etc/prom-certs/root-cert.pem", - certFile: "/etc/prom-certs/cert-chain.pem", - keyFile: "/etc/prom-certs/key.pem", - insecureSkipVerify: true, - }; - const endpoints: ServiceMonitorEndpoint[] = sm.Raw.spec.endpoints || []; - endpoints.forEach(endpoint => { - endpoint.scheme = ServiceMonitorScheme.HTTPS; - endpoint.tlsConfig = tlsConfig; - }); - sm.Raw.spec.endpoints = endpoints; - } - }); - -/** - * Mutate a pod monitor to exclude it from mTLS metrics with `exempt` scrapeClass - */ -When(PrometheusPodMonitor) - .IsCreatedOrUpdated() - .Mutate(async pm => { - if (pm.Raw.spec === undefined || pm.Raw.spec.scrapeClass != undefined) { - return; - } - - // Add an exempt scrape class if explicitly opted out via annotation OR targeting a non-istio-injected namespace - if (pm.Raw.metadata?.annotations?.["uds/skip-mutate"] || !(await isIstioInjected(pm))) { - log.info( - `Mutating scrapeClass to exempt PodMonitor ${pm.Raw.metadata?.name} from default scrapeClass mTLS config`, - ); - pm.Raw.spec.scrapeClass = "exempt"; - - return; - } else { - log.info(`Patching pod monitor ${pm.Raw.metadata?.name} for mTLS metrics`); - const endpoints: PodMonitorEndpoint[] = pm.Raw.spec.podMetricsEndpoints || []; - endpoints.forEach(endpoint => { - endpoint.scheme = PodMonitorScheme.HTTPS; - }); - pm.Raw.spec.podMetricsEndpoints = endpoints; - } - }); - -// This assumes istio-injection == strict mTLS due to complexity around mTLS lookup -async function isIstioInjected(monitor: PrometheusServiceMonitor | PrometheusPodMonitor) { - // If monitor allows any namespace assume istio injection - if (monitor.Raw.spec?.namespaceSelector?.any) { - return true; - } - - const namespaces = monitor.Raw.spec?.namespaceSelector?.matchNames || [ - monitor.Raw.metadata?.namespace, - ] || ["default"]; - - for (const ns of namespaces) { - const namespace = await K8s(kind.Namespace).Get(ns); - if (namespace.metadata?.labels && namespace.metadata.labels["istio-injection"] === "enabled") { - return true; - } - } - - return false; -} diff --git a/src/prometheus-stack/chart/templates/istio-monitor.yaml b/src/prometheus-stack/chart/templates/istio-monitor.yaml index f2871b10b..e82a0d23e 100644 --- a/src/prometheus-stack/chart/templates/istio-monitor.yaml +++ b/src/prometheus-stack/chart/templates/istio-monitor.yaml @@ -4,8 +4,6 @@ kind: PodMonitor metadata: name: envoy-stats-monitor namespace: istio-system - annotations: - uds/skip-mutate: "true" spec: selector: matchExpressions: diff --git a/src/prometheus-stack/chart/templates/peerauthentication/prometheus-operator-pa.yaml b/src/prometheus-stack/chart/templates/peerauthentication/prometheus-operator-pa.yaml deleted file mode 100644 index eb1c14d0e..000000000 --- a/src/prometheus-stack/chart/templates/peerauthentication/prometheus-operator-pa.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Capabilities.APIVersions.Has "security.istio.io/v1beta1" }} -apiVersion: "security.istio.io/v1beta1" -kind: PeerAuthentication -metadata: - name: prometheus-operator-webhook - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - app: kube-prometheus-stack-operator - mtls: - mode: STRICT - portLevelMtls: - # Allow webhooks to operate permissive since ingress originates from the nodes - "10250": - mode: PERMISSIVE -{{- end }} diff --git a/src/prometheus-stack/chart/templates/prometheus-pod-monitor.yaml b/src/prometheus-stack/chart/templates/prometheus-pod-monitor.yaml index 29f2827c2..51e17961d 100644 --- a/src/prometheus-stack/chart/templates/prometheus-pod-monitor.yaml +++ b/src/prometheus-stack/chart/templates/prometheus-pod-monitor.yaml @@ -4,8 +4,6 @@ kind: PodMonitor metadata: name: prometheus-pod-monitor namespace: monitoring - annotations: - uds/skip-mutate: "true" spec: selector: matchLabels: diff --git a/src/prometheus-stack/values/values.yaml b/src/prometheus-stack/values/values.yaml index fe6f21d26..60c9dcf11 100644 --- a/src/prometheus-stack/values/values.yaml +++ b/src/prometheus-stack/values/values.yaml @@ -24,23 +24,7 @@ prometheus: prometheusSpec: enableFeatures: - remote-write-receiver - additionalConfig: - scrapeClasses: - - name: istio-certs - default: true - tlsConfig: - caFile: /etc/prom-certs/root-cert.pem - certFile: /etc/prom-certs/cert-chain.pem - keyFile: /etc/prom-certs/key.pem - insecureSkipVerify: true - - name: exempt podMetadata: - annotations: - proxy.istio.io/config: | - proxyMetadata: - OUTPUT_CERTS: /etc/istio-output-certs - sidecar.istio.io/userVolumeMount: '[{"name": "istio-certs", "mountPath": "/etc/istio-output-certs"}]' - traffic.sidecar.istio.io/includeOutboundIPRanges: "" labels: app: prometheus podMonitorSelectorNilUsesHelmValues: false @@ -104,12 +88,3 @@ prometheusOperator: requests: cpu: 100m memory: 512Mi - -alertmanager: - alertmanagerSpec: - scheme: "https" - tlsConfig: - caFile: /etc/prom-certs/root-cert.pem - certFile: /etc/prom-certs/cert-chain.pem - insecureSkipVerify: true - keyFile: /etc/prom-certs/key.pem From 9939355d8189debd424e4a896b4e8b3977524ce7 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Fri, 23 Aug 2024 12:23:50 -0600 Subject: [PATCH 09/22] wip: ambient opt in --- .../chart/templates/uds-package.yaml | 1 + src/grafana/chart/templates/uds-package.yaml | 1 + src/keycloak/chart/templates/istio-admin.yaml | 134 +++++++++--------- src/keycloak/chart/templates/uds-package.yaml | 1 + src/loki/chart/templates/uds-package.yaml | 1 + .../chart/templates/uds-package.yaml | 1 + .../chart/templates/uds-package.yaml | 2 +- .../operator/controllers/istio/injection.ts | 120 ++++++++++------ .../crd/generated/package-v1alpha1.ts | 4 + .../operator/crd/sources/package/v1alpha1.ts | 5 + .../reconcilers/package-reconciler.ts | 6 +- .../chart/templates/uds-package.yaml | 1 + src/promtail/chart/templates/uds-package.yaml | 1 + src/test/app-admin.yaml | 1 + src/test/app-authservice-tenant.yaml | 2 - src/test/app-tenant.yaml | 1 + src/test/chart/templates/package.yaml | 1 + src/velero/chart/templates/uds-package.yaml | 1 + 18 files changed, 167 insertions(+), 117 deletions(-) diff --git a/src/authservice/chart/templates/uds-package.yaml b/src/authservice/chart/templates/uds-package.yaml index bf1041bd3..0c55b94cd 100644 --- a/src/authservice/chart/templates/uds-package.yaml +++ b/src/authservice/chart/templates/uds-package.yaml @@ -4,6 +4,7 @@ metadata: name: authservice namespace: {{ .Release.Namespace }} spec: + istioAmbient: true network: allow: # Permit intra-namespace communication for multi-replica setup diff --git a/src/grafana/chart/templates/uds-package.yaml b/src/grafana/chart/templates/uds-package.yaml index 713a103a0..564f9b685 100644 --- a/src/grafana/chart/templates/uds-package.yaml +++ b/src/grafana/chart/templates/uds-package.yaml @@ -4,6 +4,7 @@ metadata: name: grafana namespace: {{ .Release.Namespace }} spec: + istioAmbient: true sso: - name: Grafana Dashboard clientId: uds-core-admin-grafana diff --git a/src/keycloak/chart/templates/istio-admin.yaml b/src/keycloak/chart/templates/istio-admin.yaml index 09e9319f3..a73d991da 100644 --- a/src/keycloak/chart/templates/istio-admin.yaml +++ b/src/keycloak/chart/templates/istio-admin.yaml @@ -1,69 +1,65 @@ -# {{- if .Capabilities.APIVersions.Has "security.istio.io/v1beta1" }} -# apiVersion: security.istio.io/v1beta1 -# kind: AuthorizationPolicy -# metadata: -# name: keycloak-block-admin-access-from-public-gateway -# namespace: {{ .Release.Namespace }} -# spec: -# targetRefs: -# - kind: Service -# group: "" -# name: keycloak-http -# - kind: Service -# group: "" -# name: keycloak-headless -# action: DENY -# rules: -# - to: -# - operation: -# ports: -# - "8080" -# paths: -# - "/admin*" -# - "/realms/master*" -# from: -# - source: -# notNamespaces: -# - istio-admin-gateway -# - to: -# - operation: -# ports: -# - "8080" -# paths: -# - /metrics* -# from: -# - source: -# notNamespaces: -# - istio-admin-gateway -# - monitoring -# - to: -# - operation: -# ports: -# - "8080" -# paths: -# # Never allow anonymous client registration except from the pepr-system namespace -# # This is another fallback protection, as the KC policy already blocks it -# - "/realms/{{ .Values.realm }}/clients-registrations/*" -# from: -# - source: -# notNamespaces: -# - "pepr-system" -# - when: -# - key: request.headers[istio-mtls-client-certificate] -# values: ["*"] -# to: -# - operation: -# ports: -# - "8080" -# from: -# - source: -# notNamespaces: -# - istio-tenant-gateway -# - istio-admin-gateway -# {{- range .Values.additionalGatewayNamespaces }} -# {{- if not (hasPrefix "istio-" .) }} -# {{- fail (printf "Allowed gateway namespace '%s' must start with 'istio-'" .) }} -# {{- end }} -# - {{ . }} -# {{- end }} -# {{- end }} +{{- if .Capabilities.APIVersions.Has "security.istio.io/v1beta1" }} +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: keycloak-block-admin-access-from-public-gateway + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "keycloak.selectorLabels" . | nindent 6 }} + action: DENY + rules: + - to: + - operation: + ports: + - "8080" + paths: + - "/admin*" + - "/realms/master*" + from: + - source: + notNamespaces: + - istio-admin-gateway + - to: + - operation: + ports: + - "8080" + paths: + - /metrics* + from: + - source: + notNamespaces: + - istio-admin-gateway + - monitoring + - to: + - operation: + ports: + - "8080" + paths: + # Never allow anonymous client registration except from the pepr-system namespace + # This is another fallback protection, as the KC policy already blocks it + - "/realms/{{ .Values.realm }}/clients-registrations/*" + from: + - source: + notNamespaces: + - "pepr-system" + - when: + - key: request.headers[istio-mtls-client-certificate] + values: ["*"] + to: + - operation: + ports: + - "8080" + from: + - source: + notNamespaces: + - istio-tenant-gateway + - istio-admin-gateway + {{- range .Values.additionalGatewayNamespaces }} + {{- if not (hasPrefix "istio-" .) }} + {{- fail (printf "Allowed gateway namespace '%s' must start with 'istio-'" .) }} + {{- end }} + - {{ . }} + {{- end }} +{{- end }} diff --git a/src/keycloak/chart/templates/uds-package.yaml b/src/keycloak/chart/templates/uds-package.yaml index e5919fa04..4171aa882 100644 --- a/src/keycloak/chart/templates/uds-package.yaml +++ b/src/keycloak/chart/templates/uds-package.yaml @@ -4,6 +4,7 @@ metadata: name: keycloak namespace: {{ .Release.Namespace }} spec: + istioAmbient: false monitor: - selector: app.kubernetes.io/name: keycloak diff --git a/src/loki/chart/templates/uds-package.yaml b/src/loki/chart/templates/uds-package.yaml index 8f30a3d0c..fd29caf16 100644 --- a/src/loki/chart/templates/uds-package.yaml +++ b/src/loki/chart/templates/uds-package.yaml @@ -4,6 +4,7 @@ metadata: name: loki namespace: {{ .Release.Namespace }} spec: + istioAmbient: true network: allow: # Permit intra-namespace communication for gateway -> loki read/write diff --git a/src/metrics-server/chart/templates/uds-package.yaml b/src/metrics-server/chart/templates/uds-package.yaml index b213992f3..e52705f68 100644 --- a/src/metrics-server/chart/templates/uds-package.yaml +++ b/src/metrics-server/chart/templates/uds-package.yaml @@ -4,6 +4,7 @@ metadata: name: metrics-server namespace: {{ .Release.Namespace }} spec: + istioAmbient: true network: allow: - direction: Egress diff --git a/src/neuvector/chart/templates/uds-package.yaml b/src/neuvector/chart/templates/uds-package.yaml index f9c4bd08e..498002226 100644 --- a/src/neuvector/chart/templates/uds-package.yaml +++ b/src/neuvector/chart/templates/uds-package.yaml @@ -13,7 +13,7 @@ spec: # portName: metrics # targetPort: 8068 # description: "Metrics" - + istioAmbient: true sso: - name: Neuvector clientId: uds-core-admin-neuvector diff --git a/src/pepr/operator/controllers/istio/injection.ts b/src/pepr/operator/controllers/istio/injection.ts index 1fcc78249..72a517292 100644 --- a/src/pepr/operator/controllers/istio/injection.ts +++ b/src/pepr/operator/controllers/istio/injection.ts @@ -6,43 +6,77 @@ import { UDSPackage } from "../../crd"; // configure subproject logger const log = setupLogger(Component.OPERATOR_ISTIO); -const injectionLabel = "istio-injection"; -const ambientLabel = "istio.io/dataplane-mode"; -const injectionAnnotation = "uds.dev/original-istio-injection"; +const INJECTION_LABEL = "istio-injection"; +const AMBIENT_LABEL = "istio.io/dataplane-mode"; +const originalStateAnnotation = "uds.dev/original-istio-state"; + +enum IstioState { + Injected = "injected", + Ambient = "ambient", + None = "none", +} + +function getOriginalIstioState(ns: kind.Namespace): IstioState { + // Extract labels from the namespace + const labels = ns.metadata?.labels || {}; + + // Extract the relevant label values + const originalInjectionLabelValue = labels[INJECTION_LABEL]; + const originalAmbientLabelValue = labels[AMBIENT_LABEL]; + + // Determine the original Istio state based on specific label values + let istioState: IstioState; + + if (originalInjectionLabelValue === "enabled") { + istioState = IstioState.Injected; + } else if (originalAmbientLabelValue === "ambient") { + istioState = IstioState.Ambient; + } else { + istioState = IstioState.None; + } + + return istioState; +} + +function needsKill(originalIstioState: IstioState, desiredIstioState: IstioState): boolean { + return ( + (originalIstioState !== desiredIstioState && originalIstioState === IstioState.Injected) || + desiredIstioState === IstioState.Injected + ); +} /** - * Syncs the package namespace istio-injection label and adds a label for the package name + * Syncs the package namespace istio state and adds a label for the package name * * @param pkg */ -export async function enableInjection(pkg: UDSPackage) { +export async function enableIstio(pkg: UDSPackage) { if (!pkg.metadata?.namespace || !pkg.metadata.name) { throw new Error(`Invalid Package definition, missing namespace or name`); } const sourceNS = await K8s(kind.Namespace).Get(pkg.metadata.namespace); const labels = sourceNS.metadata?.labels || {}; - const originalAmbientLabel = labels[ambientLabel]; - const originalInjectionLabel = labels[injectionLabel]; const annotations = sourceNS.metadata?.annotations || {}; const pkgKey = `uds.dev/pkg-${pkg.metadata.name}`; + const desiredIstioState = pkg.spec!.istioAmbient ? IstioState.Ambient : IstioState.Injected; // Mark the original namespace injection setting for if all packages are removed - if (!annotations[injectionAnnotation]) { - annotations[injectionAnnotation] = originalInjectionLabel || "non-existent"; + const originalIstioState = getOriginalIstioState(sourceNS); + if (!annotations[originalStateAnnotation]) { + annotations[originalStateAnnotation] = originalIstioState; } // Ensure the namespace is configured - if (!annotations[pkgKey] || originalAmbientLabel !== "ambient") { - // Ensure Istio ambient is enabled - labels[ambientLabel] = "ambient"; - delete labels[injectionLabel]; - + if (!annotations[pkgKey] || originalIstioState !== desiredIstioState) { // Add the package annotation annotations[pkgKey] = "true"; + labels[AMBIENT_LABEL] = desiredIstioState === IstioState.Ambient ? "ambient" : "none"; + labels[INJECTION_LABEL] = desiredIstioState === IstioState.Ambient ? "disabled" : "enabled"; + // Apply the updated Namespace - log.debug(`Updating namespace ${pkg.metadata.namespace} with istio ambient label`); + log.debug(`Updating namespace ${pkg.metadata.namespace} with istio labels`); await K8s(kind.Namespace).Apply( { metadata: { @@ -54,12 +88,10 @@ export async function enableInjection(pkg: UDSPackage) { { force: true }, ); - // Kill the pods if we are switching from injection to ambient - if (originalInjectionLabel === "enabled") { - log.debug( - `Attempting pod restart in ${pkg.metadata.namespace} based on istio injection label change`, - ); - await killPods(pkg.metadata.namespace, false); + // Kill the pods if we are switching istio modes + if (needsKill(originalIstioState, desiredIstioState)) { + log.debug(`Attempting pod restart in ${pkg.metadata.namespace} based on istio state change`); + await killPods(pkg.metadata.namespace, desiredIstioState); } } } @@ -76,26 +108,34 @@ export async function cleanupNamespace(pkg: UDSPackage) { const sourceNS = await K8s(kind.Namespace).Get(pkg.metadata.namespace); const labels = sourceNS.metadata?.labels || {}; - const originalInjectionLabel = labels[injectionLabel]; const annotations = sourceNS.metadata?.annotations || {}; + const originalIstioState = getOriginalIstioState(sourceNS); // Remove the package annotation delete annotations[`uds.dev/pkg-${pkg.metadata.name}`]; // If there are no more UDS Package annotations, restore the original value of the istio-injection label + const desiredIstioState = annotations[originalStateAnnotation] as IstioState; if (!Object.keys(annotations).find(key => key.startsWith("uds.dev/pkg-"))) { - labels[injectionLabel] = annotations[injectionAnnotation]; - // If the original value was non-existent, remove the label - if (labels[injectionLabel] === "non-existent") { - delete labels[injectionLabel]; + switch (desiredIstioState) { + case IstioState.Ambient: + labels[AMBIENT_LABEL] = "ambient"; + labels[INJECTION_LABEL] = "disabled"; + break; + case IstioState.Injected: + labels[AMBIENT_LABEL] = "none"; + labels[INJECTION_LABEL] = "enabled"; + break; + case IstioState.None: + labels[AMBIENT_LABEL] = "none"; + labels[INJECTION_LABEL] = "disabled"; + break; } - delete annotations[injectionAnnotation]; + delete annotations[originalStateAnnotation]; } // Apply the updated Namespace - log.debug( - `Updating namespace ${pkg.metadata.namespace}, applying original istio injection labels.`, - ); + log.debug(`Updating namespace ${pkg.metadata.namespace}, applying original istio state labels.`); await K8s(kind.Namespace).Apply( { metadata: { @@ -107,12 +147,10 @@ export async function cleanupNamespace(pkg: UDSPackage) { { force: true }, ); - // Kill the pods if we changed the value of the istio-injection label - if (originalInjectionLabel !== labels[injectionLabel]) { - log.debug( - `Attempting pod restart in ${pkg.metadata.namespace} based on istio injection label change`, - ); - await killPods(pkg.metadata.namespace, true); + // Kill the pods if we changed the istio state + if (needsKill(originalIstioState, desiredIstioState)) { + log.debug(`Attempting pod restart in ${pkg.metadata.namespace} based on istio state change`); + await killPods(pkg.metadata.namespace, desiredIstioState); } } @@ -122,7 +160,7 @@ export async function cleanupNamespace(pkg: UDSPackage) { * @param ns * @param enableInjection */ -async function killPods(ns: string, enableInjection: boolean) { +async function killPods(ns: string, desiredIstioState: string) { // Get all pods in the namespace const pods = await K8s(kind.Pod).InNamespace(ns).Get(); const groups: Record = {}; @@ -138,13 +176,13 @@ async function killPods(ns: string, enableInjection: boolean) { const foundSidecar = pod.spec?.containers?.find(c => c.name === "istio-proxy"); // If enabling injection, ignore pods that already have the istio sidecar - if (enableInjection && foundSidecar) { + if (desiredIstioState === IstioState.Injected && foundSidecar) { log.debug(`Ignoring Pod ${ns}/${pod.metadata?.name}, already has sidecar`); continue; } // If disabling injection, ignore pods that don't have the istio sidecar - if (!enableInjection && !foundSidecar) { + if (desiredIstioState !== IstioState.Injected && !foundSidecar) { log.debug(`Ignoring Pod ${ns}/${pod.metadata?.name}, injection disabled`); continue; } @@ -164,9 +202,7 @@ async function killPods(ns: string, enableInjection: boolean) { } for (const pod of group) { - log.info( - `Deleting pod ${ns}/${pod.metadata?.name} to ${enableInjection ? "enable" : "disable"} the istio sidecar`, - ); + log.info(`Deleting pod ${ns}/${pod.metadata?.name} to switch to ${desiredIstioState} mode`); await K8s(kind.Pod).Delete(pod); } } diff --git a/src/pepr/operator/crd/generated/package-v1alpha1.ts b/src/pepr/operator/crd/generated/package-v1alpha1.ts index cd0f7330c..767fdc18e 100644 --- a/src/pepr/operator/crd/generated/package-v1alpha1.ts +++ b/src/pepr/operator/crd/generated/package-v1alpha1.ts @@ -8,6 +8,10 @@ export class Package extends GenericKind { } export interface Spec { + /** + * Whether or not to enable Istio Ambient mode, defaults to non-ambient (injected) + */ + istioAmbient?: boolean; /** * Create Service or Pod Monitor configurations */ diff --git a/src/pepr/operator/crd/sources/package/v1alpha1.ts b/src/pepr/operator/crd/sources/package/v1alpha1.ts index d228a0c6b..090e61a6d 100644 --- a/src/pepr/operator/crd/sources/package/v1alpha1.ts +++ b/src/pepr/operator/crd/sources/package/v1alpha1.ts @@ -480,6 +480,11 @@ export const v1alpha1: V1CustomResourceDefinitionVersion = { }, monitor, sso, + istioAmbient: { + type: "boolean", + description: + "Whether or not to enable Istio Ambient mode, defaults to non-ambient (injected)", + }, }, } as V1JSONSchemaProps, }, diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index 568f71afa..c53e9ef5f 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -1,7 +1,7 @@ import { handleFailure, shouldSkip, updateStatus, writeEvent } from "."; import { UDSConfig } from "../../config"; import { Component, setupLogger } from "../../logger"; -import { enableInjection } from "../controllers/istio/injection"; +import { enableIstio } from "../controllers/istio/injection"; import { istioResources } from "../controllers/istio/istio-resources"; import { authservice } from "../controllers/keycloak/authservice/authservice"; import { keycloak } from "../controllers/keycloak/client-sync"; @@ -62,8 +62,8 @@ export async function packageReconciler(pkg: UDSPackage) { const netPol = await networkPolicies(pkg, namespace!); let endpoints: string[] = []; - // Update the namespace to ensure the istio-injection label is set - await enableInjection(pkg); + // Update the namespace to ensure the correct istio state + await enableIstio(pkg); // Configure SSO const ssoClients = await keycloak(pkg); diff --git a/src/prometheus-stack/chart/templates/uds-package.yaml b/src/prometheus-stack/chart/templates/uds-package.yaml index 746a08692..041251461 100644 --- a/src/prometheus-stack/chart/templates/uds-package.yaml +++ b/src/prometheus-stack/chart/templates/uds-package.yaml @@ -4,6 +4,7 @@ metadata: name: prometheus-stack namespace: {{ .Release.Namespace }} spec: + istioAmbient: true network: allow: # Permit intra-namespace communication diff --git a/src/promtail/chart/templates/uds-package.yaml b/src/promtail/chart/templates/uds-package.yaml index 1a66b8490..f1486a1bb 100644 --- a/src/promtail/chart/templates/uds-package.yaml +++ b/src/promtail/chart/templates/uds-package.yaml @@ -4,6 +4,7 @@ metadata: name: promtail namespace: {{ .Release.Namespace }} spec: + istioAmbient: true monitor: - selector: app.kubernetes.io/name: promtail diff --git a/src/test/app-admin.yaml b/src/test/app-admin.yaml index 51073dc6b..1400d5c66 100644 --- a/src/test/app-admin.yaml +++ b/src/test/app-admin.yaml @@ -11,6 +11,7 @@ metadata: name: httpbin namespace: test-admin-app spec: + istioAmbient: true network: expose: - service: httpbin diff --git a/src/test/app-authservice-tenant.yaml b/src/test/app-authservice-tenant.yaml index 53bb92198..094bff22e 100644 --- a/src/test/app-authservice-tenant.yaml +++ b/src/test/app-authservice-tenant.yaml @@ -2,8 +2,6 @@ apiVersion: v1 kind: Namespace metadata: name: authservice-test-app - # labels: - # istio.io/dataplane-mode: ambient --- apiVersion: uds.dev/v1alpha1 kind: Package diff --git a/src/test/app-tenant.yaml b/src/test/app-tenant.yaml index 3a51b7b40..eb1228e4b 100644 --- a/src/test/app-tenant.yaml +++ b/src/test/app-tenant.yaml @@ -11,6 +11,7 @@ metadata: name: test-tenant-app namespace: test-tenant-app spec: + istioAmbient: true network: expose: - service: test-tenant-app diff --git a/src/test/chart/templates/package.yaml b/src/test/chart/templates/package.yaml index 1bb8d8e76..0de4dbca4 100644 --- a/src/test/chart/templates/package.yaml +++ b/src/test/chart/templates/package.yaml @@ -4,6 +4,7 @@ metadata: name: podinfo namespace: podinfo spec: + istioAmbient: true monitor: - selector: app.kubernetes.io/name: podinfo diff --git a/src/velero/chart/templates/uds-package.yaml b/src/velero/chart/templates/uds-package.yaml index 616559ebc..ad48a9329 100644 --- a/src/velero/chart/templates/uds-package.yaml +++ b/src/velero/chart/templates/uds-package.yaml @@ -4,6 +4,7 @@ metadata: name: velero namespace: {{ .Release.Namespace }} spec: + istioAmbient: true network: allow: # Todo: wide open for now for pushing to s3 From 1d4df2841a52856c93c5e73a66231bcb7b8034b8 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Fri, 23 Aug 2024 13:00:27 -0600 Subject: [PATCH 10/22] wip: istio 1.23 --- src/istio/common/zarf.yaml | 8 +++--- src/istio/values/registry1/cni.yaml | 2 +- src/istio/values/registry1/istiod.yaml | 6 ++--- src/istio/values/registry1/ztunnel.yaml | 2 +- src/istio/values/unicorn/cni.yaml | 2 +- src/istio/values/unicorn/istiod.yaml | 6 ++--- src/istio/values/upstream/cni.yaml | 2 +- src/istio/values/upstream/istiod.yaml | 6 ++--- src/istio/values/upstream/ztunnel.yaml | 2 +- src/istio/zarf.yaml | 36 ++++++++++++------------- 10 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/istio/common/zarf.yaml b/src/istio/common/zarf.yaml index 043f732fe..e85d1cff0 100644 --- a/src/istio/common/zarf.yaml +++ b/src/istio/common/zarf.yaml @@ -10,23 +10,23 @@ components: charts: - name: base url: https://istio-release.storage.googleapis.com/charts - version: 1.22.3 + version: 1.23.0 namespace: istio-system - name: istiod url: https://istio-release.storage.googleapis.com/charts - version: 1.22.3 + version: 1.23.0 namespace: istio-system valuesFiles: - "../values/base-istiod.yaml" - name: cni url: https://istio-release.storage.googleapis.com/charts - version: 1.22.3 + version: 1.23.0 namespace: istio-system valuesFiles: - "../values/base-cni.yaml" # values for k3s/k3d cni - name: ztunnel url: https://istio-release.storage.googleapis.com/charts - version: 1.22.3 + version: 1.23.0 namespace: istio-system manifests: - name: uds-global-istio-config diff --git a/src/istio/values/registry1/cni.yaml b/src/istio/values/registry1/cni.yaml index 10ad6cb3f..574c99bba 100644 --- a/src/istio/values/registry1/cni.yaml +++ b/src/istio/values/registry1/cni.yaml @@ -1,2 +1,2 @@ cni: - image: registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.22.3 + image: registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.23.0 diff --git a/src/istio/values/registry1/istiod.yaml b/src/istio/values/registry1/istiod.yaml index 83eb42e3f..a563a84d4 100644 --- a/src/istio/values/registry1/istiod.yaml +++ b/src/istio/values/registry1/istiod.yaml @@ -1,9 +1,9 @@ pilot: - image: registry1.dso.mil/ironbank/tetrate/istio/pilot:1.22.3-tetratefips-v0 + image: registry1.dso.mil/ironbank/tetrate/istio/pilot:1.23.0-tetratefips-v0 global: proxy_init: # renovate: image=registry1.dso.mil/ironbank/tetrate/istio/proxyv2 - image: "###ZARF_REGISTRY###/ironbank/tetrate/istio/proxyv2:1.22.3-tetratefips-v0" + image: "###ZARF_REGISTRY###/ironbank/tetrate/istio/proxyv2:1.23.0-tetratefips-v0" proxy: # renovate: image=registry1.dso.mil/ironbank/tetrate/istio/proxyv2 - image: "###ZARF_REGISTRY###/ironbank/tetrate/istio/proxyv2:1.22.3-tetratefips-v0" + image: "###ZARF_REGISTRY###/ironbank/tetrate/istio/proxyv2:1.23.0-tetratefips-v0" diff --git a/src/istio/values/registry1/ztunnel.yaml b/src/istio/values/registry1/ztunnel.yaml index 2ed685368..67f1aee45 100644 --- a/src/istio/values/registry1/ztunnel.yaml +++ b/src/istio/values/registry1/ztunnel.yaml @@ -1 +1 @@ -image: registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.22.3 +image: registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.23.0 diff --git a/src/istio/values/unicorn/cni.yaml b/src/istio/values/unicorn/cni.yaml index eb819c81b..cd63ec64c 100644 --- a/src/istio/values/unicorn/cni.yaml +++ b/src/istio/values/unicorn/cni.yaml @@ -1,2 +1,2 @@ cni: - image: cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.22.3 + image: cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.23.0 diff --git a/src/istio/values/unicorn/istiod.yaml b/src/istio/values/unicorn/istiod.yaml index 9847c8f04..03511e2f8 100644 --- a/src/istio/values/unicorn/istiod.yaml +++ b/src/istio/values/unicorn/istiod.yaml @@ -1,9 +1,9 @@ pilot: - image: "cgr.dev/du-uds-defenseunicorns/istio-pilot-fips:1.22.3" + image: "cgr.dev/du-uds-defenseunicorns/istio-pilot-fips:1.23.0" global: proxy_init: # renovate: image=cgr.dev/du-uds-defenseunicorns/istio-proxy-fips - image: "###ZARF_REGISTRY###/du-uds-defenseunicorns/istio-proxy-fips:1.22.3" + image: "###ZARF_REGISTRY###/du-uds-defenseunicorns/istio-proxy-fips:1.23.0" proxy: # renovate: image=cgr.dev/du-uds-defenseunicorns/istio-proxy-fips - image: "###ZARF_REGISTRY###/du-uds-defenseunicorns/istio-proxy-fips:1.22.3" + image: "###ZARF_REGISTRY###/du-uds-defenseunicorns/istio-proxy-fips:1.23.0" diff --git a/src/istio/values/upstream/cni.yaml b/src/istio/values/upstream/cni.yaml index 9ef21c6c5..1a35743a3 100644 --- a/src/istio/values/upstream/cni.yaml +++ b/src/istio/values/upstream/cni.yaml @@ -1,2 +1,2 @@ cni: - image: docker.io/istio/install-cni:1.22.3-distroless + image: docker.io/istio/install-cni:1.23.0-distroless diff --git a/src/istio/values/upstream/istiod.yaml b/src/istio/values/upstream/istiod.yaml index 86002bfa3..89d108971 100644 --- a/src/istio/values/upstream/istiod.yaml +++ b/src/istio/values/upstream/istiod.yaml @@ -1,9 +1,9 @@ pilot: - image: "docker.io/istio/pilot:1.22.3-distroless" + image: "docker.io/istio/pilot:1.23.0-distroless" global: proxy_init: # renovate: image=docker.io/istio/proxyv2 - image: "###ZARF_REGISTRY###/istio/proxyv2:1.22.3-distroless" + image: "###ZARF_REGISTRY###/istio/proxyv2:1.23.0-distroless" proxy: # renovate: image=docker.io/istio/proxyv2 - image: "###ZARF_REGISTRY###/istio/proxyv2:1.22.3-distroless" + image: "###ZARF_REGISTRY###/istio/proxyv2:1.23.0-distroless" diff --git a/src/istio/values/upstream/ztunnel.yaml b/src/istio/values/upstream/ztunnel.yaml index 15751d1f5..9e8fe5cc3 100644 --- a/src/istio/values/upstream/ztunnel.yaml +++ b/src/istio/values/upstream/ztunnel.yaml @@ -1 +1 @@ -image: docker.io/istio/ztunnel:1.22.3-distroless +image: docker.io/istio/ztunnel:1.23.0-distroless diff --git a/src/istio/zarf.yaml b/src/istio/zarf.yaml index f12c3d4a3..43031a02a 100644 --- a/src/istio/zarf.yaml +++ b/src/istio/zarf.yaml @@ -27,10 +27,10 @@ components: valuesFiles: - "values/upstream/ztunnel.yaml" images: - - "docker.io/istio/pilot:1.22.3-distroless" - - "docker.io/istio/proxyv2:1.22.3-distroless" - - "docker.io/istio/install-cni:1.22.3-distroless" - - "docker.io/istio/ztunnel:1.22.3-distroless" + - "docker.io/istio/pilot:1.23.0-distroless" + - "docker.io/istio/proxyv2:1.23.0-distroless" + - "docker.io/istio/install-cni:1.23.0-distroless" + - "docker.io/istio/ztunnel:1.23.0-distroless" - name: istio-controlplane required: true @@ -51,12 +51,12 @@ components: # - "values/registry1/ztunnel.yaml" - "values/upstream/ztunnel.yaml" images: - - registry1.dso.mil/ironbank/tetrate/istio/proxyv2:1.22.3-tetratefips-v0 - - registry1.dso.mil/ironbank/tetrate/istio/pilot:1.22.3-tetratefips-v0 - # - registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.22.3 - - docker.io/istio/ztunnel:1.22.3-distroless - # - registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.22.3 - - docker.io/istio/install-cni:1.22.3-distroless + - registry1.dso.mil/ironbank/tetrate/istio/proxyv2:1.23.0-tetratefips-v0 + - registry1.dso.mil/ironbank/tetrate/istio/pilot:1.23.0-tetratefips-v0 + # - registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.23.0 + - docker.io/istio/ztunnel:1.23.0-distroless + # - registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.23.0 + - docker.io/istio/install-cni:1.23.0-distroless - name: istio-controlplane required: true @@ -76,18 +76,18 @@ components: valuesFiles: - "values/upstream/ztunnel.yaml" images: - - cgr.dev/du-uds-defenseunicorns/istio-pilot-fips:1.22.3 - - cgr.dev/du-uds-defenseunicorns/istio-proxy-fips:1.22.3 - - docker.io/istio/install-cni:1.22.3-distroless - # - cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.22.3 - - docker.io/istio/ztunnel:1.22.3-distroless + - cgr.dev/du-uds-defenseunicorns/istio-pilot-fips:1.23.0 + - cgr.dev/du-uds-defenseunicorns/istio-proxy-fips:1.23.0 + - docker.io/istio/install-cni:1.23.0-distroless + # - cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.23.0 + - docker.io/istio/ztunnel:1.23.0-distroless - name: istio-admin-gateway required: true charts: - name: gateway url: https://istio-release.storage.googleapis.com/charts - version: 1.22.3 + version: 1.23.0 releaseName: admin-ingressgateway namespace: istio-admin-gateway - name: uds-istio-config @@ -102,7 +102,7 @@ components: charts: - name: gateway url: https://istio-release.storage.googleapis.com/charts - version: 1.22.3 + version: 1.23.0 releaseName: tenant-ingressgateway namespace: istio-tenant-gateway - name: uds-istio-config @@ -117,7 +117,7 @@ components: charts: - name: gateway url: https://istio-release.storage.googleapis.com/charts - version: 1.22.3 + version: 1.23.0 releaseName: passthrough-ingressgateway namespace: istio-passthrough-gateway - name: uds-istio-config From 9104feed9ec431cc3c06e79770dfa520e5ee180d Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Fri, 23 Aug 2024 13:01:15 -0600 Subject: [PATCH 11/22] wip: no ambient authservice --- src/authservice/chart/templates/uds-package.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/authservice/chart/templates/uds-package.yaml b/src/authservice/chart/templates/uds-package.yaml index 0c55b94cd..921e6d527 100644 --- a/src/authservice/chart/templates/uds-package.yaml +++ b/src/authservice/chart/templates/uds-package.yaml @@ -4,7 +4,7 @@ metadata: name: authservice namespace: {{ .Release.Namespace }} spec: - istioAmbient: true + istioAmbient: false network: allow: # Permit intra-namespace communication for multi-replica setup From a3f5837e23296e81422fc8d0350ba94d3882b906 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Fri, 23 Aug 2024 14:10:16 -0600 Subject: [PATCH 12/22] wip: ztunnel egress --- .../{allow-ingress-ztunnel.ts => allow-ztunnel.ts} | 8 ++++++++ src/pepr/operator/controllers/network/policies.ts | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) rename src/pepr/operator/controllers/network/defaults/{allow-ingress-ztunnel.ts => allow-ztunnel.ts} (58%) diff --git a/src/pepr/operator/controllers/network/defaults/allow-ingress-ztunnel.ts b/src/pepr/operator/controllers/network/defaults/allow-ztunnel.ts similarity index 58% rename from src/pepr/operator/controllers/network/defaults/allow-ingress-ztunnel.ts rename to src/pepr/operator/controllers/network/defaults/allow-ztunnel.ts index 2cd069065..f1b5e70df 100644 --- a/src/pepr/operator/controllers/network/defaults/allow-ingress-ztunnel.ts +++ b/src/pepr/operator/controllers/network/defaults/allow-ztunnel.ts @@ -8,3 +8,11 @@ export const allowIngressZtunnel = (namespace: string) => remoteNamespace: "*", port: 15008, }); + +export const allowEgressZtunnel = (namespace: string) => + generate(namespace, { + direction: Direction.Egress, + description: "Ztunnel", + remoteNamespace: "*", + port: 15008, + }); diff --git a/src/pepr/operator/controllers/network/policies.ts b/src/pepr/operator/controllers/network/policies.ts index e9d7e569f..9bddb7f8e 100644 --- a/src/pepr/operator/controllers/network/policies.ts +++ b/src/pepr/operator/controllers/network/policies.ts @@ -6,7 +6,7 @@ import { getOwnerRef, purgeOrphans, sanitizeResourceName } from "../utils"; import { allowEgressDNS } from "./defaults/allow-egress-dns"; import { allowEgressIstiod } from "./defaults/allow-egress-istiod"; import { allowIngressSidecarMonitoring } from "./defaults/allow-ingress-sidecar-monitoring"; -import { allowIngressZtunnel } from "./defaults/allow-ingress-ztunnel"; +import { allowEgressZtunnel, allowIngressZtunnel } from "./defaults/allow-ztunnel"; import { defaultDenyAll } from "./defaults/default-deny-all"; import { generate } from "./generate"; @@ -33,6 +33,7 @@ export async function networkPolicies(pkg: UDSPackage, namespace: string) { // Istio rules allowEgressIstiod(namespace), allowIngressZtunnel(namespace), + allowEgressZtunnel(namespace), allowIngressSidecarMonitoring(namespace), ]; From c03ea7b4a8aaa675f8c51cb7abd795d43d80a6c1 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Fri, 23 Aug 2024 15:02:53 -0600 Subject: [PATCH 13/22] wip: pepr > istio --- packages/slim-dev/zarf.yaml | 12 ++++----- packages/standard/zarf.yaml | 12 ++++----- src/istio/common/chart/.helmignore | 23 ++++++++++++++++ src/istio/common/chart/Chart.yaml | 18 +++++++++++++ .../templates}/envoy-filters.yaml | 3 --- .../common/chart/templates/exemptions.yaml | 26 +++++++++++++++++++ .../templates}/peer-authentication.yaml | 2 -- src/istio/common/chart/values.yaml | 0 .../common/manifests/pepr-istio-config.yaml | 9 ------- src/istio/common/zarf.yaml | 9 ++++--- 10 files changed, 84 insertions(+), 30 deletions(-) create mode 100644 src/istio/common/chart/.helmignore create mode 100644 src/istio/common/chart/Chart.yaml rename src/istio/common/{manifests => chart/templates}/envoy-filters.yaml (96%) create mode 100644 src/istio/common/chart/templates/exemptions.yaml rename src/istio/common/{manifests => chart/templates}/peer-authentication.yaml (71%) create mode 100644 src/istio/common/chart/values.yaml delete mode 100644 src/istio/common/manifests/pepr-istio-config.yaml diff --git a/packages/slim-dev/zarf.yaml b/packages/slim-dev/zarf.yaml index 3956b3381..0b6e5d761 100644 --- a/packages/slim-dev/zarf.yaml +++ b/packages/slim-dev/zarf.yaml @@ -19,6 +19,12 @@ components: import: path: ../../src/prometheus-stack + # Pepr the world + - name: pepr-uds-core + required: true + import: + path: ../../src/pepr + # Istio - name: istio-controlplane required: true @@ -40,12 +46,6 @@ components: import: path: ../../src/istio - # Pepr the world - - name: pepr-uds-core - required: true - import: - path: ../../src/pepr - # Keycloak - name: keycloak required: true diff --git a/packages/standard/zarf.yaml b/packages/standard/zarf.yaml index c868e74e3..4c9a1bbb6 100644 --- a/packages/standard/zarf.yaml +++ b/packages/standard/zarf.yaml @@ -19,6 +19,12 @@ components: import: path: ../../src/prometheus-stack + # Pepr the world + - name: pepr-uds-core + required: true + import: + path: ../../src/pepr + # Istio - name: istio-controlplane required: true @@ -40,12 +46,6 @@ components: import: path: ../../src/istio - # Pepr the world - - name: pepr-uds-core - required: true - import: - path: ../../src/pepr - # Metrics Server - name: metrics-server required: false diff --git a/src/istio/common/chart/.helmignore b/src/istio/common/chart/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/src/istio/common/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/src/istio/common/chart/Chart.yaml b/src/istio/common/chart/Chart.yaml new file mode 100644 index 000000000..4271ba9c9 --- /dev/null +++ b/src/istio/common/chart/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: uds-global-istio-config +description: Istio configuration for UDS + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 diff --git a/src/istio/common/manifests/envoy-filters.yaml b/src/istio/common/chart/templates/envoy-filters.yaml similarity index 96% rename from src/istio/common/manifests/envoy-filters.yaml rename to src/istio/common/chart/templates/envoy-filters.yaml index 3e4fb540f..8d5b1d6df 100644 --- a/src/istio/common/manifests/envoy-filters.yaml +++ b/src/istio/common/chart/templates/envoy-filters.yaml @@ -1,5 +1,3 @@ ---- -# Source: istio/templates/envoyfilter.yaml apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: @@ -37,7 +35,6 @@ spec: end end --- -# Source: istio/templates/envoyfilter.yaml apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: diff --git a/src/istio/common/chart/templates/exemptions.yaml b/src/istio/common/chart/templates/exemptions.yaml new file mode 100644 index 000000000..4d5e8dbaf --- /dev/null +++ b/src/istio/common/chart/templates/exemptions.yaml @@ -0,0 +1,26 @@ +apiVersion: uds.dev/v1alpha1 +kind: Exemption +metadata: + name: istio + namespace: uds-policy-exemptions +spec: + exemptions: + - policies: + - DisallowPrivileged + - RequireNonRootUser + - RestrictSELinuxType + - RestrictHostPathWrite + - RestrictVolumeTypes + - DisallowHostNamespaces + - DisallowSELinuxOptions + - DropAllCapabilities + - RestrictCapabilities + - RestrictHostPathWrite + - RestrictHostPorts + - RestrictProcMount + - RestrictSeccomp + matcher: + namespace: istio-system + name: "^.*" + title: "istio exemptions" + description: "Istio does some privileged things" diff --git a/src/istio/common/manifests/peer-authentication.yaml b/src/istio/common/chart/templates/peer-authentication.yaml similarity index 71% rename from src/istio/common/manifests/peer-authentication.yaml rename to src/istio/common/chart/templates/peer-authentication.yaml index 37ee318fc..d25302ddf 100644 --- a/src/istio/common/manifests/peer-authentication.yaml +++ b/src/istio/common/chart/templates/peer-authentication.yaml @@ -1,5 +1,3 @@ ---- -# Source: istio/templates/bigbang/peerAuthentication.yaml apiVersion: security.istio.io/v1beta1 kind: PeerAuthentication metadata: diff --git a/src/istio/common/chart/values.yaml b/src/istio/common/chart/values.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/src/istio/common/manifests/pepr-istio-config.yaml b/src/istio/common/manifests/pepr-istio-config.yaml deleted file mode 100644 index 683fd3dc5..000000000 --- a/src/istio/common/manifests/pepr-istio-config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Have to pre-create the namespace and also patch it with the istio ambient label later because -# Helm is kind of dumb: https://github.com/helm/helm/issues/350 -kind: Namespace -apiVersion: v1 -metadata: - name: pepr-system - labels: - istio-injection: disabled - istio.io/dataplane-mode: ambient diff --git a/src/istio/common/zarf.yaml b/src/istio/common/zarf.yaml index e85d1cff0..a8b366658 100644 --- a/src/istio/common/zarf.yaml +++ b/src/istio/common/zarf.yaml @@ -12,6 +12,10 @@ components: url: https://istio-release.storage.googleapis.com/charts version: 1.23.0 namespace: istio-system + - name: uds-global-istio-config + namespace: istio-system + version: 0.1.0 + localPath: chart - name: istiod url: https://istio-release.storage.googleapis.com/charts version: 1.23.0 @@ -29,12 +33,9 @@ components: version: 1.23.0 namespace: istio-system manifests: - - name: uds-global-istio-config + - name: gateway-api-crds namespace: istio-system files: - - "manifests/envoy-filters.yaml" - - "manifests/peer-authentication.yaml" - - "manifests/pepr-istio-config.yaml" - https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml actions: onDeploy: From 31b14580234a0902b5fafce2d2cb0ef76ff95c70 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Fri, 23 Aug 2024 15:28:35 -0600 Subject: [PATCH 14/22] wip: pepr action ambient --- src/istio/common/zarf.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/istio/common/zarf.yaml b/src/istio/common/zarf.yaml index a8b366658..d51509e91 100644 --- a/src/istio/common/zarf.yaml +++ b/src/istio/common/zarf.yaml @@ -44,3 +44,7 @@ components: cmd: "./zarf tools kubectl label namespace pepr-system istio.io/dataplane-mode=ambient --overwrite" - description: "Ensure istio-injection is disabled for Pepr" cmd: "./zarf tools kubectl label namespace pepr-system istio-injection=disabled --overwrite" + - description: "Cycle Pepr to refresh connections post-ambient" + cmd: | + ./zarf tools kubectl rollout restart -n pepr-system deploy/pepr-uds-core-watcher + ./zarf tools kubectl rollout restart -n pepr-system deploy/pepr-uds-core From 8dbcd9d3f89b27d5e4eeb022196ab2000f6f0d45 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Fri, 23 Aug 2024 15:57:45 -0600 Subject: [PATCH 15/22] wip: fix upgrade, single pkg --- src/istio/Ambient.md | 25 +++++++++++++++++++++++++ src/istio/common/zarf.yaml | 6 ++++++ tasks/deploy.yaml | 6 +++--- 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 src/istio/Ambient.md diff --git a/src/istio/Ambient.md b/src/istio/Ambient.md new file mode 100644 index 000000000..ca9ac6242 --- /dev/null +++ b/src/istio/Ambient.md @@ -0,0 +1,25 @@ +## Istio Ambient + +Things we had to do: +- Trusted hosts: *.pepr-uds-core-watcher.pepr-system.svc.cluster.local + +### Benefits + +- sidecar job killing - not necessary for ambient apps +- prometheus stack setup - simplified significantly with ambient on prometheus, no mutations/cert mounting required +- able to delete a number of headless services and PERMISSIVE peer authentications +- resources? +- speed? + +### Future Work/Mysteries + +- Evaluate existing netpol L3/4, find comparable istio authorization policy implementations (also reevaluate default netpols for istiod, etc) + - Package CR support for authorization policies at L4/7 + - Can we map existing network.allow from netpol -> authpolicy +- Encountered odd behavior with kubeapi - unsure if actually fixed? +- How should we handle Istio needing exemptions now (reorder pepr/istio, exemption CRD deployed pre-core, etc) +- How do we handle mission apps with authservice (require sidecar OR figure out extAuthz with waypoint) +- Keycloak and Authservice "required" sidecars to get them functional - could this be addressed by waypoints or other config we were missing? +- Gateway API instead of Ingress Gateway + VirtualService - unsure if this is connected to some of the issues we encountered with authz +- Istio ambient may need some specific configuration depending on cluster type (we had to pass in certain values for the CNI to work on [k3s](https://istio.io/latest/docs/ambient/install/platform-prerequisites/#k3s)) +- Narrow down Istio CNI/Ztunnel exemption to what is strictly necessary diff --git a/src/istio/common/zarf.yaml b/src/istio/common/zarf.yaml index d51509e91..7951ee693 100644 --- a/src/istio/common/zarf.yaml +++ b/src/istio/common/zarf.yaml @@ -39,6 +39,12 @@ components: - https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml actions: onDeploy: + before: + - description: "Fix helm ownership metadata on upgrade" + cmd: | + ./zarf tools kubectl annotate envoyfilter -n istio-system misdirected-request meta.helm.sh/release-name=uds-global-istio-config --overwrite || true + ./zarf tools kubectl annotate envoyfilter -n istio-system remove-server-header meta.helm.sh/release-name=uds-global-istio-config --overwrite || true + ./zarf tools kubectl annotate peerauthentication -n istio-system default-istio-system meta.helm.sh/release-name=uds-global-istio-config --overwrite || true after: - description: "Ensure istio ambient is enabled for Pepr" cmd: "./zarf tools kubectl label namespace pepr-system istio.io/dataplane-mode=ambient --overwrite" diff --git a/tasks/deploy.yaml b/tasks/deploy.yaml index 17cf75e8c..833f61161 100644 --- a/tasks/deploy.yaml +++ b/tasks/deploy.yaml @@ -23,14 +23,14 @@ tasks: - name: single-package actions: + - description: "Deploy the Pepr Module" + cmd: | + uds zarf package deploy build/zarf-package-pepr-uds-core-${UDS_ARCH}.tar.zst --confirm --no-progress --set UDS_SINGLE_TEST=true --set PEPR_SERVICE_MONITORS=false - description: "Deploy the Istio package, if UDS_PKG != istio" cmd: | if [ "${UDS_PKG}" != "istio" ]; then uds zarf package deploy build/zarf-package-uds-core-istio-${UDS_ARCH}.tar.zst --confirm --no-progress --components '*' fi - - description: "Deploy the Pepr Module" - cmd: | - uds zarf package deploy build/zarf-package-pepr-uds-core-${UDS_ARCH}.tar.zst --confirm --no-progress --set UDS_SINGLE_TEST=true --set PEPR_SERVICE_MONITORS=false - description: "Deploy the requested Zarf Package (must set UDS_PKG environment variable)" cmd: uds zarf package deploy build/zarf-package-uds-core-${UDS_PKG}-${UDS_ARCH}.tar.zst --confirm --no-progress --components '*' From 844f492209c9edf0c2985d685e0e32fe3bdbd915 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Fri, 23 Aug 2024 16:27:37 -0600 Subject: [PATCH 16/22] wip: docs, fixes --- .github/actions/setup/action.yaml | 2 +- src/istio/Ambient.md | 24 +++++++++++++++++------- src/istio/common/zarf.yaml | 1 + 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index 14154d48e..3339300fb 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -25,7 +25,7 @@ runs: - name: Install k3d shell: bash - run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v5.6.0 bash + run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v5.7.3 bash - name: Set up Homebrew uses: Homebrew/actions/setup-homebrew@master diff --git a/src/istio/Ambient.md b/src/istio/Ambient.md index ca9ac6242..54b258b7d 100644 --- a/src/istio/Ambient.md +++ b/src/istio/Ambient.md @@ -1,15 +1,25 @@ ## Istio Ambient -Things we had to do: -- Trusted hosts: *.pepr-uds-core-watcher.pepr-system.svc.cluster.local +### TLDR + +Migrating applications to ambient was surprisingly painless and removed more complexity than it added. At least initially we were pessimistic about the pain to migrate but were impressed how much complexity fell out when many of the quirks of istio sidecars are removed. + +We came away more optimistic about ambient and would advocate that further engineer effort be devoted to pursuing it as viable future default. ### Benefits -- sidecar job killing - not necessary for ambient apps -- prometheus stack setup - simplified significantly with ambient on prometheus, no mutations/cert mounting required -- able to delete a number of headless services and PERMISSIVE peer authentications -- resources? -- speed? +- Our current pain points with Istio sidecars (job termination and init containers mTLS traffic) become non-issues with Ambient +- The prometheus stack setup with mTLS metrics can be simplified significantly with Ambient on prometheus, we no longer require mutations or certificate mounting to properly scape endpoints. +- Ambient is able to handle direct pod addressability in a way that sidecars weren't, allowing us to remove some workarounds previously required (headless services) +- By removing the sidecars from most workloads we are able to reduce the resource footprint, especially for large scale clusters with lots of workloads on top of core +- Speed of startup as well as pod communications is increased due to the removal of sidecars (there is no longer a bottleneck to communications and pods do not have to wait on sidecars during startup) + +### Interesting Notes + +- Traffic to keycloak from Pepr originated from a "different" host, requiring a new trusted host policy in Keycloak for `*.pepr-uds-core-watcher.pepr-system.svc.cluster.local` (better than the original 127.0.0.6) +- A number of PERMISSIVE peer authentications we used for "Kube API" -> svc traffic (webhooks and api services) seem to be unnecessary with ambient +- Switching to Ambient requires a few new Istio components which do not have (working) images in Ironbank or Chainguard +- Using L7 features in Ambient mode depends on adoption of K8s Gateway API specification ### Future Work/Mysteries diff --git a/src/istio/common/zarf.yaml b/src/istio/common/zarf.yaml index 7951ee693..e15cc8a70 100644 --- a/src/istio/common/zarf.yaml +++ b/src/istio/common/zarf.yaml @@ -41,6 +41,7 @@ components: onDeploy: before: - description: "Fix helm ownership metadata on upgrade" + mute: true cmd: | ./zarf tools kubectl annotate envoyfilter -n istio-system misdirected-request meta.helm.sh/release-name=uds-global-istio-config --overwrite || true ./zarf tools kubectl annotate envoyfilter -n istio-system remove-server-header meta.helm.sh/release-name=uds-global-istio-config --overwrite || true From 9cd6edf8ac203233fbc2321708fb50625081294b Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Tue, 5 Nov 2024 13:51:23 -0700 Subject: [PATCH 17/22] fix: dupe pepr during rebase [skip ci] --- packages/base/zarf.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/base/zarf.yaml b/packages/base/zarf.yaml index fff970858..80126a7fd 100644 --- a/packages/base/zarf.yaml +++ b/packages/base/zarf.yaml @@ -23,12 +23,6 @@ components: import: path: ../../src/prometheus-stack - # Pepr the world - - name: pepr-uds-core - required: true - import: - path: ../../src/pepr - # Istio - name: istio-controlplane required: true From f97e07ba49ead154b7e984bf7e193fcee738a1a3 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Tue, 5 Nov 2024 14:06:11 -0700 Subject: [PATCH 18/22] fix: versions [skip ci] --- src/istio/values/registry1/cni.yaml | 2 +- src/istio/values/registry1/ztunnel.yaml | 2 +- src/istio/values/unicorn/cni.yaml | 2 +- src/istio/values/upstream/cni.yaml | 2 +- src/istio/values/upstream/istiod.yaml | 6 +++--- src/istio/values/upstream/ztunnel.yaml | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/istio/values/registry1/cni.yaml b/src/istio/values/registry1/cni.yaml index 6d20a9aa9..903661a30 100644 --- a/src/istio/values/registry1/cni.yaml +++ b/src/istio/values/registry1/cni.yaml @@ -2,4 +2,4 @@ # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial cni: - image: registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.23.0 + image: registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.23.2 diff --git a/src/istio/values/registry1/ztunnel.yaml b/src/istio/values/registry1/ztunnel.yaml index 647316bd0..ea959838b 100644 --- a/src/istio/values/registry1/ztunnel.yaml +++ b/src/istio/values/registry1/ztunnel.yaml @@ -1,4 +1,4 @@ # Copyright 2024 Defense Unicorns # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial -image: registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.23.0 +image: registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.23.2 diff --git a/src/istio/values/unicorn/cni.yaml b/src/istio/values/unicorn/cni.yaml index 6be296672..f89b1405f 100644 --- a/src/istio/values/unicorn/cni.yaml +++ b/src/istio/values/unicorn/cni.yaml @@ -2,4 +2,4 @@ # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial cni: - image: cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.23.0 + image: cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.23.2 diff --git a/src/istio/values/upstream/cni.yaml b/src/istio/values/upstream/cni.yaml index ba15d8864..b6401b06c 100644 --- a/src/istio/values/upstream/cni.yaml +++ b/src/istio/values/upstream/cni.yaml @@ -2,4 +2,4 @@ # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial cni: - image: docker.io/istio/install-cni:1.23.0-distroless + image: docker.io/istio/install-cni:1.23.2-distroless diff --git a/src/istio/values/upstream/istiod.yaml b/src/istio/values/upstream/istiod.yaml index 63c22fa7a..800d39f62 100644 --- a/src/istio/values/upstream/istiod.yaml +++ b/src/istio/values/upstream/istiod.yaml @@ -2,11 +2,11 @@ # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial pilot: - image: "docker.io/istio/pilot:1.23.0-distroless" + image: "docker.io/istio/pilot:1.23.2-distroless" global: proxy_init: # renovate: image=docker.io/istio/proxyv2 - image: "###ZARF_REGISTRY###/istio/proxyv2:1.23.0-distroless" + image: "###ZARF_REGISTRY###/istio/proxyv2:1.23.2-distroless" proxy: # renovate: image=docker.io/istio/proxyv2 - image: "###ZARF_REGISTRY###/istio/proxyv2:1.23.0-distroless" + image: "###ZARF_REGISTRY###/istio/proxyv2:1.23.2-distroless" diff --git a/src/istio/values/upstream/ztunnel.yaml b/src/istio/values/upstream/ztunnel.yaml index c1b48e7aa..289759dd7 100644 --- a/src/istio/values/upstream/ztunnel.yaml +++ b/src/istio/values/upstream/ztunnel.yaml @@ -1,4 +1,4 @@ # Copyright 2024 Defense Unicorns # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial -image: docker.io/istio/ztunnel:1.23.0-distroless +image: docker.io/istio/ztunnel:1.23.2-distroless From b480e35e850261743f6b4136b52cc7454795657d Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Tue, 5 Nov 2024 15:25:58 -0700 Subject: [PATCH 19/22] fix: imports [ci skip] --- pepr.ts | 4 ---- src/pepr/operator/reconcilers/package-reconciler.ts | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/pepr.ts b/pepr.ts index 18493e612..083d76b92 100644 --- a/pepr.ts +++ b/pepr.ts @@ -16,7 +16,6 @@ import { Policy } from "./src/pepr/operator/crd"; import { registerCRDs } from "./src/pepr/operator/crd/register"; import { patches } from "./src/pepr/patches"; import { policies, startExemptionWatch } from "./src/pepr/policies"; -import { prometheus } from "./src/pepr/prometheus"; const log = setupLogger(Component.STARTUP); @@ -36,9 +35,6 @@ const log = setupLogger(Component.STARTUP); // Istio service mesh istio, - // Prometheus monitoring stack - prometheus, - // Patches for specific components patches, ]); diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index a1abf9301..25d39a50b 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -6,7 +6,7 @@ import { handleFailure, shouldSkip, updateStatus, writeEvent } from "."; import { UDSConfig } from "../../config"; import { Component, setupLogger } from "../../logger"; -import { cleanupNamespace } from "../controllers/istio/injection"; +import { cleanupNamespace, enableIstio } from "../controllers/istio/injection"; import { istioResources } from "../controllers/istio/istio-resources"; import { authservice, @@ -72,7 +72,7 @@ export async function packageReconciler(pkg: UDSPackage) { let endpoints: string[] = []; // Update the namespace to ensure the istio-injection label is set - await enableInjection(pkg); + await enableIstio(pkg); let ssoClients = new Map(); let authserviceClients: string[] = []; From 4a5ef92de3573a286ceb6f202e92f90c4fee7aff Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Tue, 5 Nov 2024 15:32:27 -0700 Subject: [PATCH 20/22] chore: ordering fix [ci skip] --- packages/base/zarf.yaml | 12 ++++++------ packages/standard/zarf.yaml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/base/zarf.yaml b/packages/base/zarf.yaml index 80126a7fd..3df978342 100644 --- a/packages/base/zarf.yaml +++ b/packages/base/zarf.yaml @@ -23,6 +23,12 @@ components: import: path: ../../src/prometheus-stack + # Pepr the world + - name: pepr-uds-core + required: true + import: + path: ../../src/pepr + # Istio - name: istio-controlplane required: true @@ -43,9 +49,3 @@ components: required: false import: path: ../../src/istio - - # Pepr the world - - name: pepr-uds-core - required: true - import: - path: ../../src/pepr diff --git a/packages/standard/zarf.yaml b/packages/standard/zarf.yaml index b8568b4c7..1c4d3fd78 100644 --- a/packages/standard/zarf.yaml +++ b/packages/standard/zarf.yaml @@ -22,6 +22,12 @@ components: import: path: ../base + # Pepr the world + - name: pepr-uds-core + required: true + import: + path: ../base + # Istio - name: istio-controlplane required: true @@ -43,12 +49,6 @@ components: import: path: ../base - # Pepr the world - - name: pepr-uds-core - required: true - import: - path: ../base - # Metrics Server - name: metrics-server required: false From 7b9eaae272f9f672264b0578a9f8dcc7a790b52c Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Tue, 5 Nov 2024 16:11:29 -0700 Subject: [PATCH 21/22] wip: track broken images [ci skip] --- src/istio/values/unicorn/cni.yaml | 2 +- src/istio/zarf.yaml | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/istio/values/unicorn/cni.yaml b/src/istio/values/unicorn/cni.yaml index f89b1405f..360b88039 100644 --- a/src/istio/values/unicorn/cni.yaml +++ b/src/istio/values/unicorn/cni.yaml @@ -2,4 +2,4 @@ # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial cni: - image: cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.23.2 + image: cgr.dev/du-uds-defenseunicorns/istio-install-cni:1.23.2 diff --git a/src/istio/zarf.yaml b/src/istio/zarf.yaml index 6e9d2b960..496e58993 100644 --- a/src/istio/zarf.yaml +++ b/src/istio/zarf.yaml @@ -73,16 +73,17 @@ components: - "values/unicorn/istiod.yaml" - name: cni valuesFiles: - # - "values/unicorn/cni.yaml" - - "values/upstream/cni.yaml" + - "values/unicorn/cni.yaml" - name: ztunnel valuesFiles: - "values/upstream/ztunnel.yaml" images: - cgr.dev/du-uds-defenseunicorns/istio-pilot-fips:1.23.2 - cgr.dev/du-uds-defenseunicorns/istio-proxy-fips:1.23.2 - - docker.io/istio/install-cni:1.23.2-distroless - # - cgr.dev/du-uds-defenseunicorns/istio-install-cni-fips:1.23.2 + # Chainguard's install-cni-fips is not working right now, issue submitted + - cgr.dev/du-uds-defenseunicorns/istio-install-cni:1.23.2 + # Chainguard does not have the ztunnel image currently, but upstream is 0 CVE + # It is not currently FIPS though, and the IB TID FIPS image is amd64 only - docker.io/istio/ztunnel:1.23.2-distroless - name: istio-admin-gateway From a1201e93160022548f45be07a4db4dc0db8a72a8 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 6 Nov 2024 09:04:27 -0700 Subject: [PATCH 22/22] wip: working images [ci skip] --- src/istio/values/registry1/cni.yaml | 2 +- src/istio/values/registry1/ztunnel.yaml | 2 +- src/istio/zarf.yaml | 14 ++++++-------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/istio/values/registry1/cni.yaml b/src/istio/values/registry1/cni.yaml index 903661a30..22719e89b 100644 --- a/src/istio/values/registry1/cni.yaml +++ b/src/istio/values/registry1/cni.yaml @@ -2,4 +2,4 @@ # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial cni: - image: registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.23.2 + image: registry1.dso.mil/ironbank/opensource/istio/install-cni:1.23.3 diff --git a/src/istio/values/registry1/ztunnel.yaml b/src/istio/values/registry1/ztunnel.yaml index ea959838b..c719b11b6 100644 --- a/src/istio/values/registry1/ztunnel.yaml +++ b/src/istio/values/registry1/ztunnel.yaml @@ -1,4 +1,4 @@ # Copyright 2024 Defense Unicorns # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial -image: registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.23.2 +image: registry1.dso.mil/ironbank/tetrate/istio/ztunnel:1.23.2-tetratefips-v0 diff --git a/src/istio/zarf.yaml b/src/istio/zarf.yaml index 496e58993..82a4b902c 100644 --- a/src/istio/zarf.yaml +++ b/src/istio/zarf.yaml @@ -47,19 +47,17 @@ components: - "values/registry1/istiod.yaml" - name: cni valuesFiles: - # - "values/registry1/cni.yaml" - - "values/upstream/cni.yaml" + - "values/registry1/cni.yaml" - name: ztunnel valuesFiles: - # - "values/registry1/ztunnel.yaml" - - "values/upstream/ztunnel.yaml" + - "values/registry1/ztunnel.yaml" images: - registry1.dso.mil/ironbank/tetrate/istio/proxyv2:1.23.2-tetratefips-v0 - registry1.dso.mil/ironbank/tetrate/istio/pilot:1.23.2-tetratefips-v0 - # - registry1.dso.mil/ironbank/opensource/istio/ztunnel:1.23.2 - # - registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.23.2 - - docker.io/istio/install-cni:1.23.2-distroless - - docker.io/istio/ztunnel:1.23.2-distroless + - registry1.dso.mil/ironbank/tetrate/istio/ztunnel:1.23.2-tetratefips-v0 + # Tetrate's install-cni image is out of date currently in Ironbank but could be swapped in when updated + - registry1.dso.mil/ironbank/opensource/istio/install-cni:1.23.3 + # - registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.22.6-tetratefips-v0 - name: istio-controlplane required: true