feat: add istio ambient

.github/actions/setup/action.yaml

@@ -6,13 +6,13 @@ name: "Setup Environment"
 description: "UDS Environment Setup"
 inputs:
   ghToken:
-    description: 'GITHUB_TOKEN'
+    description: "GITHUB_TOKEN"
     required: true
   registry1Username:
-    description: 'IRON_BANK_ROBOT_USERNAME'
+    description: "IRON_BANK_ROBOT_USERNAME"
     required: true
   registry1Password:
-    description: 'IRON_BANK_ROBOT_PASSWORD'
+    description: "IRON_BANK_ROBOT_PASSWORD"
     required: true
   chainguardIdentity:
     description: "ID for Chainguard Identity"

packages/base/zarf.yaml

@@ -23,6 +23,12 @@ components:
     import:
       path: ../../src/prometheus-stack
 
+  # Pepr the world
+  - name: pepr-uds-core
+    required: true
+    import:
+      path: ../../src/pepr
+
   # Istio
   - name: istio-controlplane
     required: true
@@ -43,9 +49,3 @@ components:
     required: false
     import:
       path: ../../src/istio
-
-  # Pepr the world
-  - name: pepr-uds-core
-    required: true
-    import:
-      path: ../../src/pepr

packages/standard/zarf.yaml

@@ -22,6 +22,12 @@ components:
     import:
       path: ../base
 
+  # Pepr the world
+  - name: pepr-uds-core
+    required: true
+    import:
+      path: ../base
+
   # Istio
   - name: istio-controlplane
     required: true
@@ -43,12 +49,6 @@ components:
     import:
       path: ../base
 
-  # Pepr the world
-  - name: pepr-uds-core
-    required: true
-    import:
-      path: ../base
-
   # Metrics Server
   - name: metrics-server
     required: false

pepr.ts

@@ -16,7 +16,6 @@ import { Policy } from "./src/pepr/operator/crd";
 import { registerCRDs } from "./src/pepr/operator/crd/register";
 import { patches } from "./src/pepr/patches";
 import { policies, startExemptionWatch } from "./src/pepr/policies";
-import { prometheus } from "./src/pepr/prometheus";
 
 const log = setupLogger(Component.STARTUP);
 
@@ -36,9 +35,6 @@ const log = setupLogger(Component.STARTUP);
     // Istio service mesh
     istio,
 
-    // Prometheus monitoring stack
-    prometheus,
-
     // Patches for specific components
     patches,
   ]);

src/authservice/chart/templates/uds-package.yaml

@@ -7,6 +7,7 @@ metadata:
   name: authservice
   namespace: {{ .Release.Namespace }}
 spec:
+  istioAmbient: false
   network:
     allow:
       # Permit intra-namespace communication for multi-replica setup

src/grafana/chart/templates/uds-package.yaml

@@ -7,6 +7,7 @@ metadata:
   name: grafana
   namespace: {{ .Release.Namespace }}
 spec:
+  istioAmbient: true
   sso:
     - name: Grafana Dashboard
       clientId: uds-core-admin-grafana

src/istio/Ambient.md

@@ -0,0 +1,35 @@
+## Istio Ambient
+
+### TLDR
+
+Migrating applications to ambient was surprisingly painless and removed more complexity than it added. At least initially we were pessimistic about the pain to migrate but were impressed how much complexity fell out when many of the quirks of istio sidecars are removed.
+
+We came away more optimistic about ambient and would advocate that further engineer effort be devoted to pursuing it as viable future default.
+
+### Benefits
+
+- Our current pain points with Istio sidecars (job termination and init containers mTLS traffic) become non-issues with Ambient
+- The prometheus stack setup with mTLS metrics can be simplified significantly with Ambient on prometheus, we no longer require mutations or certificate mounting to properly scape endpoints.
+- Ambient is able to handle direct pod addressability in a way that sidecars weren't, allowing us to remove some workarounds previously required (headless services)
+- By removing the sidecars from most workloads we are able to reduce the resource footprint, especially for large scale clusters with lots of workloads on top of core
+- Speed of startup as well as pod communications is increased due to the removal of sidecars (there is no longer a bottleneck to communications and pods do not have to wait on sidecars during startup)
+
+### Interesting Notes
+
+- Traffic to keycloak from Pepr originated from a "different" host, requiring a new trusted host policy in Keycloak for `*.pepr-uds-core-watcher.pepr-system.svc.cluster.local` (better than the original 127.0.0.6)
+- A number of PERMISSIVE peer authentications we used for "Kube API" -> svc traffic (webhooks and api services) seem to be unnecessary with ambient
+- Switching to Ambient requires a few new Istio components which do not have (working) images in Ironbank or Chainguard
+- Using L7 features in Ambient mode depends on adoption of K8s Gateway API specification
+
+### Future Work/Mysteries
+
+- Evaluate existing netpol L3/4, find comparable istio authorization policy implementations (also reevaluate default netpols for istiod, etc)
+  - Package CR support for authorization policies at L4/7
+  - Can we map existing network.allow from netpol -> authpolicy
+- Encountered odd behavior with kubeapi - unsure if actually fixed?
+- How should we handle Istio needing exemptions now (reorder pepr/istio, exemption CRD deployed pre-core, etc)
+- How do we handle mission apps with authservice (require sidecar OR figure out extAuthz with waypoint)
+- Keycloak and Authservice "required" sidecars to get them functional - could this be addressed by waypoints or other config we were missing?
+- Gateway API instead of Ingress Gateway + VirtualService - unsure if this is connected to some of the issues we encountered with authz
+- Istio ambient may need some specific configuration depending on cluster type (we had to pass in certain values for the CNI to work on [k3s](https://istio.io/latest/docs/ambient/install/platform-prerequisites/#k3s))
+- Narrow down Istio CNI/Ztunnel exemption to what is strictly necessary

src/istio/common/chart/templates/envoy-filters.yaml

@@ -40,7 +40,6 @@ spec:
                 end
               end
 ---
-# Source: istio/templates/envoyfilter.yaml
 apiVersion: networking.istio.io/v1alpha3
 kind: EnvoyFilter
 metadata:

src/istio/common/chart/templates/exemptions.yaml

@@ -0,0 +1,29 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+apiVersion: uds.dev/v1alpha1
+kind: Exemption
+metadata:
+  name: istio
+  namespace: uds-policy-exemptions
+spec:
+  exemptions:
+    - policies:
+        - DisallowPrivileged
+        - RequireNonRootUser
+        - RestrictSELinuxType
+        - RestrictHostPathWrite
+        - RestrictVolumeTypes
+        - DisallowHostNamespaces
+        - DisallowSELinuxOptions
+        - DropAllCapabilities
+        - RestrictCapabilities
+        - RestrictHostPathWrite
+        - RestrictHostPorts
+        - RestrictProcMount
+        - RestrictSeccomp
+      matcher:
+        namespace: istio-system
+        name: "^.*"
+      title: "istio exemptions"
+      description: "Istio does some privileged things"

src/istio/common/zarf.yaml

@@ -15,28 +15,41 @@ components:
         url: https://istio-release.storage.googleapis.com/charts
         version: 1.23.2
         namespace: istio-system
+      - name: uds-global-istio-config
+        namespace: istio-system
+        version: 0.1.0
+        localPath: chart
       - name: istiod
         url: https://istio-release.storage.googleapis.com/charts
         version: 1.23.2
         namespace: istio-system
         valuesFiles:
-          - "../values/values.yaml"
-      - name: uds-global-istio-config
+          - "../values/base-istiod.yaml"
+      - name: cni
+        url: https://istio-release.storage.googleapis.com/charts
+        version: 1.23.2
+        namespace: istio-system
+        valuesFiles:
+          - "../values/base-cni.yaml" # values for k3s/k3d cni
+      - name: ztunnel
+        url: https://istio-release.storage.googleapis.com/charts
+        version: 1.23.2
         namespace: istio-system
-        version: 0.1.0
-        localPath: chart
     actions:
       onDeploy:
         before:
-          - description: "Fix helm ownership if necessary for clean helm upgrade"
+          - description: "Fix helm ownership metadata on upgrade"
             mute: true
             cmd: |
-              ./zarf tools kubectl annotate EnvoyFilter misdirected-request -n istio-system meta.helm.sh/release-name=uds-global-istio-config --overwrite || true
-              ./zarf tools kubectl annotate EnvoyFilter remove-server-header -n istio-system meta.helm.sh/release-name=uds-global-istio-config --overwrite || true
-              ./zarf tools kubectl annotate PeerAuthentication default-istio-system -n istio-system meta.helm.sh/release-name=uds-global-istio-config --overwrite || true
-              ./zarf tools kubectl annotate PeerAuthentication permissive-pepr-webhook -n pepr-system meta.helm.sh/release-name=uds-global-istio-config --overwrite || true
-              ./zarf tools kubectl annotate PeerAuthentication permissive-pepr-webhook-watcher -n pepr-system meta.helm.sh/release-name=uds-global-istio-config --overwrite || true
+              ./zarf tools kubectl annotate envoyfilter -n istio-system misdirected-request meta.helm.sh/release-name=uds-global-istio-config --overwrite || true
+              ./zarf tools kubectl annotate envoyfilter -n istio-system remove-server-header meta.helm.sh/release-name=uds-global-istio-config --overwrite || true
+              ./zarf tools kubectl annotate peerauthentication -n istio-system default-istio-system meta.helm.sh/release-name=uds-global-istio-config --overwrite || true
         after:
-          - description: "Ensure istio-injection is enabled for Pepr"
-            mute: true
-            cmd: "./zarf tools kubectl label namespace pepr-system istio-injection=enabled --overwrite"
+          - description: "Ensure istio ambient is enabled for Pepr"
+            cmd: "./zarf tools kubectl label namespace pepr-system istio.io/dataplane-mode=ambient --overwrite"
+          - description: "Ensure istio-injection is disabled for Pepr"
+            cmd: "./zarf tools kubectl label namespace pepr-system istio-injection=disabled --overwrite"
+          - description: "Cycle Pepr to refresh connections post-ambient"
+            cmd: |
+              ./zarf tools kubectl rollout restart -n pepr-system deploy/pepr-uds-core-watcher
+              ./zarf tools kubectl rollout restart -n pepr-system deploy/pepr-uds-core

src/istio/tasks.yaml

@@ -7,21 +7,21 @@ tasks:
       - description: Validate the Istio Admin Gateway
         wait:
           cluster:
-            kind: Gateway
+            kind: gateways.networking.istio.io
             name: admin-gateway
             namespace: istio-admin-gateway
 
       - description: Validate the Istio Passthrough Gateway
         wait:
           cluster:
-            kind: Gateway
+            kind: gateways.networking.istio.io
             name: passthrough-gateway
             namespace: istio-passthrough-gateway
 
       - description: Validate the Istio Tenant Gateway
         wait:
           cluster:
-            kind: Gateway
+            kind: gateways.networking.istio.io
             name: tenant-gateway
             namespace: istio-tenant-gateway
 

src/istio/values/base-cni.yaml

@@ -0,0 +1,8 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+profile: ambient
+
+cni:
+  cniConfDir: /var/lib/rancher/k3s/agent/etc/cni/net.d
+  cniBinDir: /bin/

src/istio/values/values.yaml → src/istio/values/base-istiod.yaml

@@ -1,6 +1,8 @@
 # Copyright 2024 Defense Unicorns
 # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
 
+profile: ambient
+
 meshConfig:
   accessLogFile: /dev/stdout
   pathNormalization:

src/istio/values/registry1/cni.yaml

@@ -0,0 +1,5 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+cni:
+  image: registry1.dso.mil/ironbank/opensource/istio/install-cni:1.23.3

src/istio/values/registry1/ztunnel.yaml

@@ -0,0 +1,4 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+image: registry1.dso.mil/ironbank/tetrate/istio/ztunnel:1.23.2-tetratefips-v0

src/istio/values/unicorn/cni.yaml

@@ -0,0 +1,5 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+cni:
+  image: cgr.dev/du-uds-defenseunicorns/istio-install-cni:1.23.2

src/istio/values/upstream/cni.yaml

@@ -0,0 +1,5 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+cni:
+  image: docker.io/istio/install-cni:1.23.2-distroless

src/istio/values/upstream/istiod.yaml

@@ -0,0 +1,12 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+pilot:
+  image: "docker.io/istio/pilot:1.23.2-distroless"
+global:
+  proxy_init:
+    # renovate: image=docker.io/istio/proxyv2
+    image: "###ZARF_REGISTRY###/istio/proxyv2:1.23.2-distroless"
+  proxy:
+    # renovate: image=docker.io/istio/proxyv2
+    image: "###ZARF_REGISTRY###/istio/proxyv2:1.23.2-distroless"

src/istio/values/upstream/ztunnel.yaml

@@ -0,0 +1,4 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+image: docker.io/istio/ztunnel:1.23.2-distroless

src/istio/zarf.yaml

@@ -22,10 +22,18 @@ components:
     charts:
       - name: istiod
         valuesFiles:
-          - "values/upstream-values.yaml"
+          - "values/upstream/istiod.yaml"
+      - name: cni
+        valuesFiles:
+          - "values/upstream/cni.yaml"
+      - name: ztunnel
+        valuesFiles:
+          - "values/upstream/ztunnel.yaml"
     images:
       - "docker.io/istio/pilot:1.23.2-distroless"
       - "docker.io/istio/proxyv2:1.23.2-distroless"
+      - "docker.io/istio/install-cni:1.23.2-distroless"
+      - "docker.io/istio/ztunnel:1.23.2-distroless"
 
   - name: istio-controlplane
     required: true
@@ -36,10 +44,20 @@ components:
     charts:
       - name: istiod
         valuesFiles:
-          - "values/registry1-values.yaml"
+          - "values/registry1/istiod.yaml"
+      - name: cni
+        valuesFiles:
+          - "values/registry1/cni.yaml"
+      - name: ztunnel
+        valuesFiles:
+          - "values/registry1/ztunnel.yaml"
     images:
       - registry1.dso.mil/ironbank/tetrate/istio/proxyv2:1.23.2-tetratefips-v0
       - registry1.dso.mil/ironbank/tetrate/istio/pilot:1.23.2-tetratefips-v0
+      - registry1.dso.mil/ironbank/tetrate/istio/ztunnel:1.23.2-tetratefips-v0
+      # Tetrate's install-cni image is out of date currently in Ironbank but could be swapped in when updated
+      - registry1.dso.mil/ironbank/opensource/istio/install-cni:1.23.3
+      # - registry1.dso.mil/ironbank/tetrate/istio/install-cni:1.22.6-tetratefips-v0
 
   - name: istio-controlplane
     required: true
@@ -50,10 +68,21 @@ components:
     charts:
       - name: istiod
         valuesFiles:
-          - "values/unicorn-values.yaml"
+          - "values/unicorn/istiod.yaml"
+      - name: cni
+        valuesFiles:
+          - "values/unicorn/cni.yaml"
+      - name: ztunnel
+        valuesFiles:
+          - "values/upstream/ztunnel.yaml"
     images:
       - cgr.dev/du-uds-defenseunicorns/istio-pilot-fips:1.23.2
       - cgr.dev/du-uds-defenseunicorns/istio-proxy-fips:1.23.2
+      # Chainguard's install-cni-fips is not working right now, issue submitted
+      - cgr.dev/du-uds-defenseunicorns/istio-install-cni:1.23.2
+      # Chainguard does not have the ztunnel image currently, but upstream is 0 CVE
+      # It is not currently FIPS though, and the IB TID FIPS image is amd64 only
+      - docker.io/istio/ztunnel:1.23.2-distroless
 
   - name: istio-admin-gateway
     required: true