chore: add prometheus, loki, and vector e2e testing (#939)

## Description
Add E2E prometheus, loki, and vector testing. 

## Related Issues

Fixes #824 #823 #822 

## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [x] Other (security config, docs update, etc)

## Checklist before merging

- [x] Test, docs, adr added or updated as needed
- [x] [Contributor
Guide](https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md)
followed

---------

Co-authored-by: Rob Ferguson <rjferguson21@gmail.com>
Co-authored-by: Noah <40781376+noahpb@users.noreply.github.com>
Co-authored-by: Noah Birrer <noah@defenseunicorns.com>
Co-authored-by: Micah Nagel <micah.nagel@defenseunicorns.com>
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

.codespellrc

@@ -1,6 +1,6 @@
 # Lint Codespell configurations
 [codespell]
 skip = .codespellrc,.git,node_modules,build,dist,*.zst,CHANGELOG.md,.playwright,.terraform
-ignore-words-list = NotIn,AKS,LICENS,aks
+ignore-words-list = NotIn,AKS,LICENS,aks,afterAll
 enable-colors =
 check-hidden =

.eslintrc.json

@@ -21,7 +21,7 @@
     "node_modules",
     "dist",
     "jest.*.js",
-    "e2e/"
+    "test/playwright/"
   ],
   "root": true,
   "rules": {

.github/actions/save-logs/action.yaml

@@ -40,7 +40,7 @@ runs:
     - name: Move Playwright Artifacts
       run: |
         sudo mkdir -p /tmp/playwright
-        sudo mv e2e/playwright/.playwright/* /tmp/playwright || true
+        sudo mv test/playwright/.playwright/* /tmp/playwright || true
       shell: bash
 
     - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3

.yamllint

@@ -7,7 +7,7 @@ ignore:
   - 'k3d/local/manifests/metallb/metallb-native.yaml'
   - '**/.terraform/**'
   - '**/chart/templates/**'
-  - 'node_modules/**'
+  - '**/node_modules/**'
   - 'dist/**'
   - 'src/pepr/uds-operator-config/templates/**'
   - '.codespellrc'

packages/logging/tasks.yaml

@@ -10,3 +10,4 @@ tasks:
     actions:
       - task: loki:validate
       - task: vector:validate
+      - task: loki:e2e-test

packages/monitoring/tasks.yaml

@@ -11,3 +11,4 @@ tasks:
       - task: prometheus:validate
       - task: grafana:validate
       - task: grafana:e2e-test
+      - task: prometheus:e2e-test

renovate.json

@@ -74,7 +74,7 @@
       "commitMessageTopic": "grafana"
     },
     {
-      "matchFileNames": ["e2e/playwright/**", ".github/**", "bundles/**", "tasks/*.yaml", ".vscode/settings.json", "src/test/**"],
+      "matchFileNames": ["test/**", ".github/**", "bundles/**", "tasks/*.yaml", ".vscode/settings.json", "src/test/**"],
       "groupName": "support-deps",
       "commitMessageTopic": "support dependencies"
     },

src/authservice/tasks.yaml

@@ -11,3 +11,7 @@ tasks:
             name: "app.kubernetes.io/name=authservice"
             namespace: authservice
             condition: Ready
+
+  - name: e2e-test
+    actions:
+      - description: "Run Authservice E2E tests"

src/grafana/tasks.yaml

@@ -45,4 +45,4 @@ tasks:
             npm ci && \
             npx playwright test grafana.test.ts \
             "
-        dir: e2e/playwright
+        dir: test/playwright

src/istio/tasks.yaml

@@ -24,3 +24,7 @@ tasks:
             kind: Gateway
             name: tenant-gateway
             namespace: istio-tenant-gateway
+
+  - name: e2e-test
+    actions:
+      - description: "Run Istio E2E tests"

src/keycloak/tasks.yaml

@@ -44,3 +44,7 @@ tasks:
   - name: regenerate-test-pki
     actions:
       - task: config:regenerate-test-pki
+
+  - name: e2e-test
+    actions:
+      - description: "Run Keycloak E2E tests"

src/kiali/tasks.yaml

@@ -8,3 +8,7 @@ tasks:
         cmd: "echo Replace Me"
         # wait:
         #   cluster:
+
+  - name: e2e-test
+    actions:
+      - description: "Run Kiali E2E tests"

src/loki/tasks.yaml

@@ -24,3 +24,10 @@ tasks:
             name: app.kubernetes.io/component=gateway
             namespace: loki
             condition: Ready
+
+  - name: e2e-test
+    actions:
+      - description: "Run Loki E2E tests"
+        cmd: |
+            npm ci && npx jest --testPathPattern loki*
+        dir: test/jest

src/metrics-server/tasks.yaml

@@ -16,3 +16,7 @@ tasks:
             kind: APIService
             name: v1beta1.metrics.k8s.io
             condition: available
+
+  - name: e2e-test
+    actions:
+      - description: "Run Metrics-Server E2E tests"

src/neuvector/tasks.yaml

@@ -38,3 +38,7 @@ tasks:
             protocol: https
             address: neuvector.admin.uds.dev
             code: 200
+
+  - name: e2e-test
+    actions:
+      - description: "Run Neuvector E2E tests"

src/pepr/istio/index.ts

@@ -84,7 +84,7 @@ When(a.Pod)
           log.info(`Terminated sidecar for ${key}`);
         } catch (err) {
           log.error({ err }, `Failed to terminate the sidecar for ${key}`);
-
+        } finally {
           // Remove the pod from the seen list
           inProgress.delete(key);
         }

src/pepr/tasks.yaml

@@ -50,3 +50,7 @@ tasks:
               # Remove the json and go files
               rm -f "$CRD-crd.json"
           done
+
+  - name: e2e-test
+    actions:
+      - description: "Run Pepr E2E tests"

src/prometheus-stack/tasks.yaml

@@ -41,3 +41,10 @@ tasks:
   #       cmd: "npx kubernetes-fluent-client crd https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.77.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml src/pepr/operator/crd/generated/prometheus"
   #     - description: Pepr Format
   #       cmd: "npx pepr format"
+
+  - name: e2e-test
+    actions:
+      - description: "Run Prometheus-Stack E2E tests"
+        cmd: |
+            npm ci && npx jest --testPathPattern prometheus*
+        dir: test/jest

src/tempo/tasks.yaml

@@ -8,3 +8,7 @@ tasks:
         cmd: "echo Replace Me"
         # wait:
         #   cluster:
+
+  - name: e2e-test
+    actions:
+      - description: "Run Tempo E2E tests"

src/test/tasks.yaml

@@ -148,3 +148,7 @@ tasks:
 
       - description: Remove the test resources
         cmd: "uds zarf package remove build/zarf-package-uds-core-test-apps-*.zst --confirm --no-progress"
+
+  - name: e2e-test
+    actions:
+      - description: "Run Test Apps E2E tests"

src/vector/tasks.yaml

@@ -11,3 +11,6 @@ tasks:
             name: app.kubernetes.io/name=vector
             namespace: vector
             condition: Ready
+  - name: e2e-test
+    actions:
+      - description: "Run Vector E2E tests"

src/vector/values/values.yaml

@@ -5,6 +5,10 @@
 role: "Agent"
 
 customConfig:
+  api:
+    enabled: false
+    address: 0.0.0.0:8686
+    playground: false
   data_dir: /var/lib/vector
   # Ensure e2e delivery of events
   acknowledgements:

src/velero/tasks.yaml

@@ -83,3 +83,7 @@ tasks:
             fi
           fi
           echo "Status is Complete! Great success!"
+
+  - name: e2e-test
+    actions:
+      - description: "Run Velero E2E tests"

tasks/test.yaml

@@ -62,14 +62,18 @@ tasks:
         # Self-reference this task file to avoid https://github.com/defenseunicorns/maru-runner/issues/144
         cmd: uds run -f tasks/test.yaml common-setup:create-doug-user --set KEYCLOAK_GROUP="/UDS Core/Admin" # Adds the test doug user
       - description: "Run Playwright E2E tests for all packages"
-        dir: e2e/playwright
+        dir: test/playwright
         cmd: |
           # renovate: datasource=docker depName=mcr.microsoft.com/playwright versioning=docker
           docker run --rm --ipc=host --net=host -e FULL_CORE="true" --mount type=bind,source="$(pwd)",target=/app mcr.microsoft.com/playwright:v1.48.2-noble sh -c " \
             cd app && \
             npm ci && \
             npx playwright test \
             "
+      - description: "Run E2E Tests"
+        cmd: |
+          npm ci
+          npx jest test/jest/
 
   - name: uds-core
     description: "Build and test UDS Core"

test/jest/forward.ts

@@ -0,0 +1,122 @@
+/**
+ * Copyright 2024 Defense Unicorns
+ * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+ */
+
+import * as k8s from "@kubernetes/client-node";
+import { K8s, kind } from "kubernetes-fluent-client";
+import * as net from "net";
+
+const kc = new k8s.KubeConfig();
+const forward = new k8s.PortForward(kc);
+kc.loadFromDefault();
+
+interface ForwardResult {
+  server: net.Server;
+  url: string;
+}
+
+// Utility function to get an available random port within a range
+async function getAvailablePort(min = 1024, max = 65535): Promise<number> {
+  let port: number;
+  let isAvailable = false;
+
+  while (!isAvailable) {
+    port = Math.floor(Math.random() * (max - min + 1)) + min;
+    isAvailable = await new Promise<boolean>(resolve => {
+      const server = net.createServer();
+
+      server.once("error", () => resolve(false)); // Port is in use
+      server.once("listening", () => {
+        server.close(() => resolve(true)); // Port is available
+      });
+
+      server.listen(port, "127.0.0.1");
+    });
+  }
+
+  return port!;
+}
+
+export async function getPodFromService(svc: string, namespace: string): Promise<string> {
+  try {
+    const service = await K8s(kind.Service).InNamespace(namespace).Get(svc);
+    const labelSelector = service.spec?.selector;
+
+    if (!labelSelector) {
+      throw new Error(`No label selectors found for service: ${svc}`);
+    }
+
+    let podsQuery = K8s(kind.Pod).InNamespace(namespace);
+    for (const key in labelSelector) {
+      podsQuery = podsQuery.WithLabel(key, labelSelector[key]);
+    }
+
+    const pods = await podsQuery.Get();
+    if (pods.items.length === 0) {
+      throw new Error(`No pods found for service: ${svc}`);
+    }
+
+    return pods.items[0].metadata!.name!;
+  } catch (err) {
+    // Type guard to check if `err` is an instance of `Error`
+    if (err instanceof Error) {
+      throw new Error(`Failed to get pod from service ${svc}: ${err.message}`);
+    } else {
+      throw new Error(`Unknown error occurred while fetching pod from service ${svc}`);
+    }
+  }
+}
+
+export async function getForward(
+  service: string,
+  namespace: string,
+  port: number,
+): Promise<ForwardResult> {
+  try {
+    const podName = await getPodFromService(service, namespace);
+    const randomPort = await getAvailablePort(3000, 65535);
+
+    return await new Promise<ForwardResult>((resolve, reject) => {
+      const server = net.createServer(socket => {
+        // Explicitly ignore the promise with `void` to avoid eslint no-floating-promises error
+        void forward.portForward(namespace, podName, [port], socket, null, socket);
+      });
+
+      server.listen(randomPort, "127.0.0.1", () => {
+        resolve({ server, url: `http://localhost:${randomPort}` });
+      });
+
+      server.on("error", err => {
+        if (err instanceof Error) {
+          reject(new Error(`Error binding to port ${randomPort}: ${err.message}`));
+        } else {
+          reject(new Error(`Unknown error occurred while binding to port ${randomPort}`));
+        }
+      });
+    });
+  } catch (err) {
+    if (err instanceof Error) {
+      throw new Error(`Failed to setup port forwarding for service ${service}: ${err.message}`);
+    } else {
+      throw new Error(
+        `Unknown error occurred while setting up port forwarding for service ${service}`,
+      );
+    }
+  }
+}
+
+export function closeForward(server: net.Server): Promise<void> {
+  return new Promise((resolve, reject) => {
+    server.close(err => {
+      // Type guard to check if `err` is an instance of `Error`
+      if (err instanceof Error) {
+        reject(new Error(`Failed to close server: ${err.message}`));
+      } else if (err) {
+        reject(new Error("Unknown error occurred while closing the server"));
+      } else {
+        resolve();
+      }
+    });
+  });
+}