From 1bde4ccf302864b0c38d093742ca683b96cebe89 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 14 May 2024 12:34:03 -0600 Subject: [PATCH] feat: add `expose` service entry for internal cluster traffic (#356) ## Description This adds a service entry to allow traffic to stay inside the cluster and enable things like proper network policies when clients need to access this endpoint. ## Related Issue Fixes #N/A ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [X] Other (security config, docs update, etc) ## Checklist before merging - [X] Test, docs, adr added or updated as needed - [X] [Contributor Guide Steps](https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md)(https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md#submitting-a-pull-request) followed --------- Co-authored-by: Chance <139784371+UnicornChance@users.noreply.github.com> Co-authored-by: Micah Nagel --- .eslintrc.json | 10 +- .../templates/secret-admin-password.yaml | 2 +- .../chart/templates/secret-postgresql.yaml | 2 +- src/pepr/operator/README.md | 8 +- .../controllers/istio/istio-resources.ts | 92 ++++++++++ .../controllers/istio/service-entry.spec.ts | 53 ++++++ .../controllers/istio/service-entry.ts | 79 +++++++++ .../controllers/istio/virtual-service.spec.ts | 112 ++++++++++++ .../controllers/istio/virtual-service.ts | 165 +++++++----------- .../monitoring/service-monitor.spec.ts | 15 +- .../controllers/monitoring/service-monitor.ts | 17 +- .../generated/istio/serviceentry-v1beta1.ts | 142 +++++++++++++++ src/pepr/operator/crd/index.ts | 16 +- .../crd/validators/package-validator.ts | 3 +- .../reconcilers/package-reconciler.ts | 6 +- tasks.yaml | 4 + 16 files changed, 600 insertions(+), 126 deletions(-) create mode 100644 src/pepr/operator/controllers/istio/istio-resources.ts create mode 100644 src/pepr/operator/controllers/istio/service-entry.spec.ts create mode 100644 src/pepr/operator/controllers/istio/service-entry.ts create mode 100644 src/pepr/operator/controllers/istio/virtual-service.spec.ts create mode 100644 src/pepr/operator/crd/generated/istio/serviceentry-v1beta1.ts diff --git a/.eslintrc.json b/.eslintrc.json index 85f4b187a..dcbc9ca6c 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -14,5 +14,13 @@ "root": true, "rules": { "@typescript-eslint/no-floating-promises": ["error"] - } + }, + "overrides": [ + { + "files": [ "src/pepr/operator/crd/generated/**/*.ts", "src/pepr/operator/crd/generated/*.ts" ], + "rules": { + "@typescript-eslint/no-explicit-any": "off" + } + } + ] } diff --git a/src/keycloak/chart/templates/secret-admin-password.yaml b/src/keycloak/chart/templates/secret-admin-password.yaml index 04e42d5a8..59307eb32 100644 --- a/src/keycloak/chart/templates/secret-admin-password.yaml +++ b/src/keycloak/chart/templates/secret-admin-password.yaml @@ -13,7 +13,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ $secretName }} - namespace: {{ .Release.Namespace }} + namespace: {{ .Release.Namespace }} labels: {{- include "keycloak.labels" . | nindent 4 }} type: Opaque diff --git a/src/keycloak/chart/templates/secret-postgresql.yaml b/src/keycloak/chart/templates/secret-postgresql.yaml index 771ad3a16..e0af8d089 100644 --- a/src/keycloak/chart/templates/secret-postgresql.yaml +++ b/src/keycloak/chart/templates/secret-postgresql.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ include "keycloak.fullname" . }}-postgresql - namespace: {{ .Release.Namespace }} + namespace: {{ .Release.Namespace }} labels: {{- include "keycloak.labels" . | nindent 4 }} type: Opaque diff --git a/src/pepr/operator/README.md b/src/pepr/operator/README.md index fe152f391..737828f60 100644 --- a/src/pepr/operator/README.md +++ b/src/pepr/operator/README.md @@ -8,7 +8,7 @@ The UDS Operator manages the lifecycle of UDS Package CRs and their correspondin - establishing default-deny ingress/egress network policies - creating a layered allow-list based approach on top of the default deny network policies including some basic defaults such as Istio requirements and DNS egress - providing targeted remote endpoints network policies such as `KubeAPI` and `CloudMetadata` to make policies more DRY and provide dynamic bindings where a static definition is not possible -- creating Istio Virtual Services & related ingress gateway network policies +- creating Istio Virtual Services, Service Entries & related ingress gateway network policies #### Exemption @@ -25,7 +25,7 @@ metadata: namespace: grafana spec: network: - # Expose rules generate Istio VirtualServices and related network policies + # Expose rules generate Istio VirtualServices, ServiceEntries and related network policies expose: - service: grafana selector: @@ -196,8 +196,8 @@ graph TD G -->|Yes| H["Log: Skipping pkg"] G -->|No| I["Update pkg status to Phase.Pending"] I --> J{"Check if Istio is installed"} - J -->|Yes| K["Add injection label, process expose CRs for Virtual Services"] - J -->|No| L["Skip Virtual Service Creation"] + J -->|Yes| K["Add injection label, process expose CRs for Istio Resources"] + J -->|No| L["Skip Istio Resource Creation"] K --> M["Create default network policies in namespace"] L --> M M --> N["Process allow CRs for network policies"] diff --git a/src/pepr/operator/controllers/istio/istio-resources.ts b/src/pepr/operator/controllers/istio/istio-resources.ts new file mode 100644 index 000000000..84406067a --- /dev/null +++ b/src/pepr/operator/controllers/istio/istio-resources.ts @@ -0,0 +1,92 @@ +import { K8s, Log } from "pepr"; + +import { IstioVirtualService, IstioServiceEntry, UDSPackage } from "../../crd"; +import { getOwnerRef } from "../utils"; +import { generateVirtualService } from "./virtual-service"; +import { generateServiceEntry } from "./service-entry"; + +/** + * Creates a VirtualService and ServiceEntry for each exposed service in the package + * + * @param pkg + * @param namespace + */ +export async function istioResources(pkg: UDSPackage, namespace: string) { + const pkgName = pkg.metadata!.name!; + const generation = (pkg.metadata?.generation ?? 0).toString(); + const ownerRefs = getOwnerRef(pkg); + + // Get the list of exposed services + const exposeList = pkg.spec?.network?.expose ?? []; + + // Create a Set of processed hosts (to maintain uniqueness) + const hosts = new Set(); + + // Track which ServiceEntries we've created + const serviceEntryNames: Map = new Map(); + + // Iterate over each exposed service + for (const expose of exposeList) { + // Generate a VirtualService for this `expose` entry + const vsPayload = generateVirtualService(expose, namespace, pkgName, generation, ownerRefs); + + Log.debug(vsPayload, `Applying VirtualService ${vsPayload.metadata?.name}`); + + // Apply the VirtualService and force overwrite any existing policy + await K8s(IstioVirtualService).Apply(vsPayload, { force: true }); + + vsPayload.spec!.hosts!.forEach(h => hosts.add(h)); + + // Generate a ServiceEntry for this `expose` entry + const sePayload = generateServiceEntry(expose, namespace, pkgName, generation, ownerRefs); + + // If we have already made a ServiceEntry with this name, skip (i.e. if advancedHTTP was used) + if (serviceEntryNames.get(sePayload.metadata!.name!)) { + continue; + } + + Log.debug(sePayload, `Applying ServiceEntry ${sePayload.metadata?.name}`); + + // Apply the ServiceEntry and force overwrite any existing policy + await K8s(IstioServiceEntry).Apply(sePayload, { force: true }); + + serviceEntryNames.set(sePayload.metadata!.name!, true); + } + + // Get all related VirtualServices in the namespace + const virtualServices = await K8s(IstioVirtualService) + .InNamespace(namespace) + .WithLabel("uds/package", pkgName) + .Get(); + + // Find any orphaned VirtualServices (not matching the current generation) + const orphanedVS = virtualServices.items.filter( + vs => vs.metadata?.labels?.["uds/generation"] !== generation, + ); + + // Delete any orphaned VirtualServices + for (const vs of orphanedVS) { + Log.debug(vs, `Deleting orphaned VirtualService ${vs.metadata!.name}`); + await K8s(IstioVirtualService).Delete(vs); + } + + // Get all related ServiceEntries in the namespace + const serviceEntries = await K8s(IstioServiceEntry) + .InNamespace(namespace) + .WithLabel("uds/package", pkgName) + .Get(); + + // Find any orphaned ServiceEntries (not matching the current generation) + const orphanedSE = serviceEntries.items.filter( + se => se.metadata?.labels?.["uds/generation"] !== generation, + ); + + // Delete any orphaned ServiceEntries + for (const se of orphanedSE) { + Log.debug(se, `Deleting orphaned ServiceEntry ${se.metadata!.name}`); + await K8s(IstioServiceEntry).Delete(se); + } + + // Return the list of unique hostnames + return [...hosts]; +} diff --git a/src/pepr/operator/controllers/istio/service-entry.spec.ts b/src/pepr/operator/controllers/istio/service-entry.spec.ts new file mode 100644 index 000000000..caeb1be1c --- /dev/null +++ b/src/pepr/operator/controllers/istio/service-entry.spec.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "@jest/globals"; +import { UDSConfig } from "../../../config"; +import { generateServiceEntry } from "./service-entry"; +import { Expose, Gateway, IstioLocation, IstioResolution } from "../../crd"; + +describe("test generate service entry", () => { + const ownerRefs = [ + { + apiVersion: "uds.dev/v1alpha1", + kind: "Package", + name: "test", + uid: "f50120aa-2713-4502-9496-566b102b1174", + }, + ]; + + const host = "test"; + const port = 8080; + const service = "test-service"; + + const namespace = "test"; + const pkgName = "test"; + const generation = "1"; + + it("should create a simple ServiceEntry object", () => { + const expose: Expose = { + host, + port, + service, + }; + + const payload = generateServiceEntry(expose, namespace, pkgName, generation, ownerRefs); + + expect(payload).toBeDefined(); + expect(payload.metadata?.name).toEqual(`${pkgName}-${Gateway.Tenant}-${host}`); + expect(payload.metadata?.namespace).toEqual(namespace); + + expect(payload.spec?.hosts).toBeDefined(); + expect(payload.spec!.hosts![0]).toEqual(`${host}.${UDSConfig.domain}`); + + expect(payload.spec!.location).toEqual(IstioLocation.MeshInternal); + expect(payload.spec!.resolution).toEqual(IstioResolution.DNS); + + expect(payload.spec?.ports).toBeDefined(); + expect(payload.spec!.ports![0].name).toEqual("https"); + expect(payload.spec!.ports![0].number).toEqual(443); + expect(payload.spec!.ports![0].protocol).toEqual("HTTPS"); + + expect(payload.spec?.endpoints).toBeDefined(); + expect(payload.spec!.endpoints![0].address).toEqual( + `${Gateway.Tenant}-ingressgateway.istio-${Gateway.Tenant}-gateway.svc.cluster.local`, + ); + }); +}); diff --git a/src/pepr/operator/controllers/istio/service-entry.ts b/src/pepr/operator/controllers/istio/service-entry.ts new file mode 100644 index 000000000..b326a99d0 --- /dev/null +++ b/src/pepr/operator/controllers/istio/service-entry.ts @@ -0,0 +1,79 @@ +import { UDSConfig } from "../../../config"; +import { V1OwnerReference } from "@kubernetes/client-node"; +import { + Expose, + Gateway, + IstioServiceEntry, + IstioLocation, + IstioResolution, + IstioPort, + IstioEndpoint, +} from "../../crd"; +import { sanitizeResourceName } from "../utils"; + +/** + * Creates a ServiceEntry for each exposed service in the package + * + * @param pkg + * @param namespace + */ +export function generateServiceEntry( + expose: Expose, + namespace: string, + pkgName: string, + generation: string, + ownerRefs: V1OwnerReference[], +) { + const { gateway = Gateway.Tenant, host } = expose; + + const name = generateSEName(pkgName, expose); + + // For the admin gateway, we need to add the path prefix + const domain = (gateway === Gateway.Admin ? "admin." : "") + UDSConfig.domain; + + // Append the domain to the host + const fqdn = `${host}.${domain}`; + + const serviceEntryPort: IstioPort = { + name: "https", + number: 443, + protocol: "HTTPS", + }; + + const serviceEntryEndpoint: IstioEndpoint = { + // Map the gateway (admin, passthrough or tenant) to the ServiceEntry + address: `${gateway}-ingressgateway.istio-${gateway}-gateway.svc.cluster.local`, + }; + + const payload: IstioServiceEntry = { + metadata: { + name, + namespace, + labels: { + "uds/package": pkgName, + "uds/generation": generation, + }, + // Use the CR as the owner ref for each ServiceEntry + ownerReferences: ownerRefs, + }, + spec: { + // Append the UDS Domain to the host + hosts: [fqdn], + location: IstioLocation.MeshInternal, + resolution: IstioResolution.DNS, + ports: [serviceEntryPort], + endpoints: [serviceEntryEndpoint], + }, + }; + + return payload; +} + +export function generateSEName(pkgName: string, expose: Expose) { + const { gateway = Gateway.Tenant, host } = expose; + + // Ensure the resource name is valid + const name = sanitizeResourceName(`${pkgName}-${gateway}-${host}`); + + return name; +} diff --git a/src/pepr/operator/controllers/istio/virtual-service.spec.ts b/src/pepr/operator/controllers/istio/virtual-service.spec.ts new file mode 100644 index 000000000..890d0c12b --- /dev/null +++ b/src/pepr/operator/controllers/istio/virtual-service.spec.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from "@jest/globals"; +import { UDSConfig } from "../../../config"; +import { generateVirtualService } from "./virtual-service"; +import { Expose, Gateway } from "../../crd"; + +describe("test generate virtual service", () => { + const ownerRefs = [ + { + apiVersion: "uds.dev/v1alpha1", + kind: "Package", + name: "test", + uid: "f50120aa-2713-4502-9496-566b102b1174", + }, + ]; + + const host = "test"; + const port = 8080; + const service = "test-service"; + + const namespace = "test"; + const pkgName = "test"; + const generation = "1"; + + it("should create a simple VirtualService object", () => { + const expose: Expose = { + host, + port, + service, + }; + + const payload = generateVirtualService(expose, namespace, pkgName, generation, ownerRefs); + + expect(payload).toBeDefined(); + expect(payload.metadata?.name).toEqual( + `${pkgName}-${Gateway.Tenant}-${host}-${port}-${service}`, + ); + expect(payload.metadata?.namespace).toEqual(namespace); + + expect(payload.spec?.hosts).toBeDefined(); + expect(payload.spec!.hosts![0]).toEqual(`${host}.${UDSConfig.domain}`); + + expect(payload.spec?.http).toBeDefined(); + expect(payload.spec!.http![0].route).toBeDefined(); + expect(payload.spec!.http![0].route![0].destination?.host).toEqual( + `${service}.${namespace}.svc.cluster.local`, + ); + expect(payload.spec!.http![0].route![0].destination?.port?.number).toEqual(port); + + expect(payload.spec?.gateways).toBeDefined(); + expect(payload.spec!.gateways![0]).toEqual( + `istio-${Gateway.Tenant}-gateway/${Gateway.Tenant}-gateway`, + ); + }); + + it("should create an admin VirtualService object", () => { + const gateway = Gateway.Admin; + const expose: Expose = { + gateway, + host, + port, + service, + }; + + const payload = generateVirtualService(expose, namespace, pkgName, generation, ownerRefs); + + expect(payload).toBeDefined(); + expect(payload.spec?.hosts).toBeDefined(); + expect(payload.spec!.hosts![0]).toEqual(`${host}.admin.${UDSConfig.domain}`); + }); + + it("should create an advancedHttp VirtualService object", () => { + const advancedHTTP = { + directResponse: { status: 404 }, + }; + const expose: Expose = { + host, + port, + service, + advancedHTTP, + }; + + const payload = generateVirtualService(expose, namespace, pkgName, generation, ownerRefs); + + expect(payload).toBeDefined(); + expect(payload.spec?.http).toBeDefined(); + expect(payload.spec!.http![0].route).not.toBeDefined(); + expect(payload.spec!.http![0].directResponse?.status).toEqual(404); + }); + + it("should create a passthrough VirtualService object", () => { + const gateway = Gateway.Passthrough; + const expose: Expose = { + gateway, + host, + port, + service, + }; + + const payload = generateVirtualService(expose, namespace, pkgName, generation, ownerRefs); + + expect(payload).toBeDefined(); + expect(payload.spec?.tls).toBeDefined(); + expect(payload.spec!.tls![0].match).toBeDefined(); + expect(payload.spec!.tls![0].match![0].port).toEqual(443); + expect(payload.spec!.tls![0].match![0].sniHosts![0]).toEqual(`${host}.${UDSConfig.domain}`); + expect(payload.spec!.tls![0].route).toBeDefined(); + expect(payload.spec!.http![0].route![0].destination?.host).toEqual( + `${service}.${namespace}.svc.cluster.local`, + ); + expect(payload.spec!.http![0].route![0].destination?.port?.number).toEqual(port); + }); +}); diff --git a/src/pepr/operator/controllers/istio/virtual-service.ts b/src/pepr/operator/controllers/istio/virtual-service.ts index 913625699..591fa691f 100644 --- a/src/pepr/operator/controllers/istio/virtual-service.ts +++ b/src/pepr/operator/controllers/istio/virtual-service.ts @@ -1,8 +1,7 @@ -import { K8s, Log } from "pepr"; - import { UDSConfig } from "../../../config"; -import { Expose, Gateway, Istio, UDSPackage } from "../../crd"; -import { getOwnerRef, sanitizeResourceName } from "../utils"; +import { V1OwnerReference } from "@kubernetes/client-node"; +import { Expose, Gateway, IstioVirtualService, IstioHTTP, IstioHTTPRoute } from "../../crd"; +import { sanitizeResourceName } from "../utils"; /** * Creates a VirtualService for each exposed service in the package @@ -10,114 +9,82 @@ import { getOwnerRef, sanitizeResourceName } from "../utils"; * @param pkg * @param namespace */ -export async function virtualService(pkg: UDSPackage, namespace: string) { - const pkgName = pkg.metadata!.name!; - const generation = (pkg.metadata?.generation ?? 0).toString(); - - // Get the list of exposed services - const exposeList = pkg.spec?.network?.expose ?? []; - - // Create a list of generated VirtualServices - const payloads: Istio.VirtualService[] = []; - - // Iterate over each exposed service - for (const expose of exposeList) { - const { gateway = Gateway.Tenant, host, port, service, advancedHTTP = {} } = expose; - - const name = generateVSName(pkg, expose); - - // For the admin gateway, we need to add the path prefix - const domain = (gateway === Gateway.Admin ? "admin." : "") + UDSConfig.domain; - - // Append the domain to the host - const fqdn = `${host}.${domain}`; - - const http: Istio.HTTP = { ...advancedHTTP }; - - // Create the route to the service - const route: Istio.HTTPRoute[] = [ - { - destination: { - // Use the service name as the host - host: `${service}.${namespace}.svc.cluster.local`, - // The CRD only uses numeric ports - port: { number: port }, - }, +export function generateVirtualService( + expose: Expose, + namespace: string, + pkgName: string, + generation: string, + ownerRefs: V1OwnerReference[], +) { + const { gateway = Gateway.Tenant, host, port, service, advancedHTTP = {} } = expose; + + const name = generateVSName(pkgName, expose); + + // For the admin gateway, we need to add the path prefix + const domain = (gateway === Gateway.Admin ? "admin." : "") + UDSConfig.domain; + + // Append the domain to the host + const fqdn = `${host}.${domain}`; + + const http: IstioHTTP = { ...advancedHTTP }; + + // Create the route to the service + const route: IstioHTTPRoute[] = [ + { + destination: { + // Use the service name as the host + host: `${service}.${namespace}.svc.cluster.local`, + // The CRD only uses numeric ports + port: { number: port }, }, - ]; + }, + ]; - if (!advancedHTTP.directResponse) { - // Create the route to the service if not using advancedHTTP.directResponse - http.route = route; - } + if (!advancedHTTP.directResponse) { + // Create the route to the service if not using advancedHTTP.directResponse + http.route = route; + } - const payload: Istio.VirtualService = { - metadata: { - name, - namespace, - labels: { - "uds/package": pkgName, - "uds/generation": generation, - }, - // Use the CR as the owner ref for each VirtualService - ownerReferences: getOwnerRef(pkg), + const payload: IstioVirtualService = { + metadata: { + name, + namespace, + labels: { + "uds/package": pkgName, + "uds/generation": generation, }, - spec: { - // Append the UDS Domain to the host - hosts: [fqdn], - // Map the gateway (admin, passthrough or tenant) to the VirtualService - gateways: [`istio-${gateway}-gateway/${gateway}-gateway`], - // Apply the route to the VirtualService - http: [http], + // Use the CR as the owner ref for each VirtualService + ownerReferences: ownerRefs, + }, + spec: { + // Append the UDS Domain to the host + hosts: [fqdn], + // Map the gateway (admin, passthrough or tenant) to the VirtualService + gateways: [`istio-${gateway}-gateway/${gateway}-gateway`], + // Apply the route to the VirtualService + http: [http], + }, + }; + + // If the gateway is the passthrough gateway, apply the TLS match + if (gateway === Gateway.Passthrough) { + payload.spec!.tls = [ + { + match: [{ port: 443, sniHosts: [fqdn] }], + route, }, - }; - - // If the gateway is the passthrough gateway, apply the TLS match - if (gateway === Gateway.Passthrough) { - payload.spec!.tls = [ - { - match: [{ port: 443, sniHosts: [fqdn] }], - route, - }, - ]; - } - - Log.debug(payload, `Applying VirtualService ${name}`); - - // Apply the VirtualService and force overwrite any existing policy - await K8s(Istio.VirtualService).Apply(payload, { force: true }); - - payloads.push(payload); - } - - // Get all related VirtualServices in the namespace - const virtualServices = await K8s(Istio.VirtualService) - .InNamespace(namespace) - .WithLabel("uds/package", pkgName) - .Get(); - - // Find any orphaned VirtualServices (not matching the current generation) - const orphanedVS = virtualServices.items.filter( - vs => vs.metadata?.labels?.["uds/generation"] !== generation, - ); - - // Delete any orphaned VirtualServices - for (const vs of orphanedVS) { - Log.debug(vs, `Deleting orphaned VirtualService ${vs.metadata!.name}`); - await K8s(Istio.VirtualService).Delete(vs); + ]; } - - // Return the list of unique hostnames - return [...new Set(payloads.map(v => v.spec!.hosts!).flat())]; + return payload; } -export function generateVSName(pkg: UDSPackage, expose: Expose) { +export function generateVSName(pkgName: string, expose: Expose) { const { gateway = Gateway.Tenant, host, port, service, description, advancedHTTP } = expose; // Ensure the resource name is valid const matchHash = advancedHTTP?.match?.flatMap(m => m.name).join("-") || ""; const nameSuffix = description || `${host}-${port}-${service}-${matchHash}`; - const name = sanitizeResourceName(`${pkg.metadata!.name}-${gateway}-${nameSuffix}`); + const name = sanitizeResourceName(`${pkgName}-${gateway}-${nameSuffix}`); return name; } diff --git a/src/pepr/operator/controllers/monitoring/service-monitor.spec.ts b/src/pepr/operator/controllers/monitoring/service-monitor.spec.ts index e4cb5cfc7..83d4fa03e 100644 --- a/src/pepr/operator/controllers/monitoring/service-monitor.spec.ts +++ b/src/pepr/operator/controllers/monitoring/service-monitor.spec.ts @@ -1,20 +1,21 @@ import { describe, expect, it } from "@jest/globals"; import { generateServiceMonitor } from "./service-monitor"; +import { Monitor } from "../../crd"; describe("test generate service monitor", () => { it("should return a valid Service Monitor object", () => { - const pkg = { - apiVersion: "uds.dev/v1alpha1", - kind: "Package", - metadata: { + const ownerRefs = [ + { + apiVersion: "uds.dev/v1alpha1", + kind: "Package", name: "test", uid: "f50120aa-2713-4502-9496-566b102b1174", }, - }; + ]; const portName = "http-metrics"; const metricsPath = "/test"; const selectorApp = "test"; - const monitor = { + const monitor: Monitor = { portName: portName, path: metricsPath, targetPort: 1234, @@ -25,7 +26,7 @@ describe("test generate service monitor", () => { const namespace = "test"; const pkgName = "test"; const generation = "1"; - const payload = generateServiceMonitor(pkg, monitor, namespace, pkgName, generation); + const payload = generateServiceMonitor(monitor, namespace, pkgName, generation, ownerRefs); expect(payload).toBeDefined(); expect(payload.metadata?.name).toEqual(`${pkgName}-${selectorApp}-${portName}`); diff --git a/src/pepr/operator/controllers/monitoring/service-monitor.ts b/src/pepr/operator/controllers/monitoring/service-monitor.ts index 7756142a6..ff2ba0713 100644 --- a/src/pepr/operator/controllers/monitoring/service-monitor.ts +++ b/src/pepr/operator/controllers/monitoring/service-monitor.ts @@ -1,7 +1,7 @@ import { K8s, Log } from "pepr"; -import { Prometheus, UDSPackage } from "../../crd"; -import { Monitor } from "../../crd/generated/package-v1alpha1"; +import { V1OwnerReference } from "@kubernetes/client-node"; +import { Prometheus, UDSPackage, Monitor } from "../../crd"; import { getOwnerRef, sanitizeResourceName } from "../utils"; /** @@ -13,6 +13,7 @@ import { getOwnerRef, sanitizeResourceName } from "../utils"; export async function serviceMonitor(pkg: UDSPackage, namespace: string) { const pkgName = pkg.metadata!.name!; const generation = (pkg.metadata?.generation ?? 0).toString(); + const ownerRefs = getOwnerRef(pkg); Log.debug(`Reconciling ServiceMonitors for ${pkgName}`); @@ -24,7 +25,7 @@ export async function serviceMonitor(pkg: UDSPackage, namespace: string) { try { for (const monitor of monitorList) { - const payload = generateServiceMonitor(pkg, monitor, namespace, pkgName, generation); + const payload = generateServiceMonitor(monitor, namespace, pkgName, generation, ownerRefs); Log.debug(payload, `Applying ServiceMonitor ${payload.metadata?.name}`); @@ -60,25 +61,25 @@ export async function serviceMonitor(pkg: UDSPackage, namespace: string) { return [...payloads.map(sm => sm.metadata!.name!)]; } -export function generateSMName(pkg: UDSPackage, monitor: Monitor) { +export function generateSMName(pkgName: string, monitor: Monitor) { const { selector, portName, description } = monitor; // Ensure the resource name is valid const nameSuffix = description || `${Object.values(selector)}-${portName}`; - const name = sanitizeResourceName(`${pkg.metadata!.name}-${nameSuffix}`); + const name = sanitizeResourceName(`${pkgName}-${nameSuffix}`); return name; } export function generateServiceMonitor( - pkg: UDSPackage, monitor: Monitor, namespace: string, pkgName: string, generation: string, + ownerRefs: V1OwnerReference[], ) { const { selector, portName } = monitor; - const name = generateSMName(pkg, monitor); + const name = generateSMName(pkgName, monitor); const payload: Prometheus.ServiceMonitor = { metadata: { name, @@ -87,7 +88,7 @@ export function generateServiceMonitor( "uds/package": pkgName, "uds/generation": generation, }, - ownerReferences: getOwnerRef(pkg), + ownerReferences: ownerRefs, }, spec: { endpoints: [ diff --git a/src/pepr/operator/crd/generated/istio/serviceentry-v1beta1.ts b/src/pepr/operator/crd/generated/istio/serviceentry-v1beta1.ts new file mode 100644 index 000000000..3ab93402f --- /dev/null +++ b/src/pepr/operator/crd/generated/istio/serviceentry-v1beta1.ts @@ -0,0 +1,142 @@ +// This file is auto-generated by kubernetes-fluent-client, do not edit manually + +import { GenericKind, RegisterKind } from "kubernetes-fluent-client"; + +export class ServiceEntry extends GenericKind { + /** + * Configuration affecting service registry. See more details at: + * https://istio.io/docs/reference/config/networking/service-entry.html + */ + spec?: Spec; + status?: { [key: string]: any }; +} + +/** + * Configuration affecting service registry. See more details at: + * https://istio.io/docs/reference/config/networking/service-entry.html + */ +export interface Spec { + /** + * The virtual IP addresses associated with the service. + */ + addresses?: string[]; + /** + * One or more endpoints associated with the service. + */ + endpoints?: Endpoint[]; + /** + * A list of namespaces to which this service is exported. + */ + exportTo?: string[]; + /** + * The hosts associated with the ServiceEntry. + */ + hosts: string[]; + /** + * Specify whether the service should be considered external to the mesh or part of the mesh. + */ + location?: Location; + /** + * The ports associated with the external service. + */ + ports?: Port[]; + /** + * Service resolution mode for the hosts. + */ + resolution?: Resolution; + /** + * If specified, the proxy will verify that the server certificate's subject alternate name + * matches one of the specified values. + */ + subjectAltNames?: string[]; + /** + * Applicable only for MESH_INTERNAL services. + */ + workloadSelector?: WorkloadSelector; +} + +export interface Endpoint { + /** + * Address associated with the network endpoint without the port. + */ + address?: string; + /** + * One or more labels associated with the endpoint. + */ + labels?: { [key: string]: string }; + /** + * The locality associated with the endpoint. + */ + locality?: string; + /** + * Network enables Istio to group endpoints resident in the same L3 domain/network. + */ + network?: string; + /** + * Set of ports associated with the endpoint. + */ + ports?: { [key: string]: number }; + /** + * The service account associated with the workload if a sidecar is present in the workload. + */ + serviceAccount?: string; + /** + * The load balancing weight associated with the endpoint. + */ + weight?: number; +} + +/** + * Specify whether the service should be considered external to the mesh or part of the mesh. + */ +export enum Location { + MeshExternal = "MESH_EXTERNAL", + MeshInternal = "MESH_INTERNAL", +} + +export interface Port { + /** + * Label assigned to the port. + */ + name: string; + /** + * A valid non-negative integer port number. + */ + number: number; + /** + * The protocol exposed on the port. + */ + protocol?: string; + /** + * The port number on the endpoint where the traffic will be received. + */ + targetPort?: number; +} + +/** + * Service resolution mode for the hosts. + */ +export enum Resolution { + DNS = "DNS", + DNSRoundRobin = "DNS_ROUND_ROBIN", + None = "NONE", + Static = "STATIC", +} + +/** + * Applicable only for MESH_INTERNAL services. + */ +export interface WorkloadSelector { + /** + * One or more labels that indicate a specific set of pods/VMs on which the configuration + * should be applied. + */ + labels?: { [key: string]: string }; +} + +RegisterKind(ServiceEntry, { + group: "networking.istio.io", + version: "v1beta1", + kind: "ServiceEntry", + plural: "serviceentries", +}); diff --git a/src/pepr/operator/crd/index.ts b/src/pepr/operator/crd/index.ts index 11982ead1..959ae0029 100644 --- a/src/pepr/operator/crd/index.ts +++ b/src/pepr/operator/crd/index.ts @@ -2,6 +2,7 @@ export { Allow, Direction, Expose, + Monitor, Gateway, Phase, Status as PkgStatus, @@ -20,5 +21,18 @@ export { Exemption as UDSExemption, } from "./generated/exemption-v1alpha1"; -export * as Istio from "./generated/istio/virtualservice-v1beta1"; +export { + VirtualService as IstioVirtualService, + HTTPRoute as IstioHTTPRoute, + HTTP as IstioHTTP, +} from "./generated/istio/virtualservice-v1beta1"; + +export { + ServiceEntry as IstioServiceEntry, + Location as IstioLocation, + Resolution as IstioResolution, + Endpoint as IstioEndpoint, + Port as IstioPort, +} from "./generated/istio/serviceentry-v1beta1"; + export * as Prometheus from "./generated/prometheus/servicemonitor-v1"; diff --git a/src/pepr/operator/crd/validators/package-validator.ts b/src/pepr/operator/crd/validators/package-validator.ts index 3cbe19251..8155bf0e5 100644 --- a/src/pepr/operator/crd/validators/package-validator.ts +++ b/src/pepr/operator/crd/validators/package-validator.ts @@ -11,6 +11,7 @@ const invalidNamespaces = ["kube-system", "kube-public", "_unknown_", "pepr-syst export async function validator(req: PeprValidateRequest) { const pkg = migrate(req.Raw); + const pkgName = pkg.metadata?.name ?? "_unknown_"; const ns = pkg.metadata?.namespace ?? "_unknown_"; if (invalidNamespaces.includes(ns)) { @@ -38,7 +39,7 @@ export async function validator(req: PeprValidateRequest) { } // Ensure the service name is unique - const name = generateVSName(req.Raw, expose); + const name = generateVSName(pkgName, expose); if (virtualServiceNames.has(name)) { return req.Deny( `The combination of characteristics of this expose entry would create a duplicate VirtualService. ` + diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index 2c500301d..4e522a14c 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -3,7 +3,7 @@ import { Log } from "pepr"; import { handleFailure, shouldSkip, updateStatus } from "."; import { UDSConfig } from "../../config"; import { enableInjection } from "../controllers/istio/injection"; -import { virtualService } from "../controllers/istio/virtual-service"; +import { istioResources } from "../controllers/istio/istio-resources"; import { keycloak } from "../controllers/keycloak/client-sync"; import { serviceMonitor } from "../controllers/monitoring/service-monitor"; import { networkPolicies } from "../controllers/network/policies"; @@ -40,8 +40,8 @@ export async function packageReconciler(pkg: UDSPackage) { // Update the namespace to ensure the istio-injection label is set await enableInjection(pkg); - // Create the VirtualService for each exposed service - endpoints = await virtualService(pkg, namespace!); + // Create the VirtualService and ServiceEntry for each exposed service + endpoints = await istioResources(pkg, namespace!); // Only configure the ServiceMonitors if not running in single test mode let monitors: string[] = []; diff --git a/tasks.yaml b/tasks.yaml index fd5f367b7..9ff6b4971 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -30,6 +30,10 @@ tasks: - description: "Deploy the Istio source package with Zarf Dev" cmd: "uds zarf dev deploy src/istio --flavor ${FLAVOR}" + # Note, this abuses the --flavor flag to only install the CRDs from this package - the "crds-only" flavor is not an explicit flavor of the package + - description: "Deploy the Prometheus-Stack source package with Zarf Dev to only install the CRDs" + cmd: "uds zarf dev deploy src/prometheus-stack --flavor crds-only" + - description: "Dev instructions" cmd: | echo "Next steps:"