From 4180d62b99b1e3d9891f8281cd0a1adb2e10ac3b Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 21 Aug 2026 13:22:05 +0200 Subject: [PATCH 1/4] test(e2e): add minimal CLI conformance profile Signed-off-by: Evan Lezar --- TESTING.md | 13 +++++++++++++ e2e/rust/Cargo.toml | 4 ++++ tasks/test.toml | 6 ++++++ 3 files changed, 23 insertions(+) diff --git a/TESTING.md b/TESTING.md index 6c0829060d..f20d1b93d9 100644 --- a/TESTING.md +++ b/TESTING.md @@ -148,6 +148,9 @@ lifecycle management, output parsing, and cleanup. Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. +- CLI conformance (`--features e2e-cli-conformance`) - a curated portable CLI + baseline selected from existing E2E test targets. The initial profile contains + the gateway smoke workflow and remains additive to the regular E2E suites. - Docker suite (`--features e2e-docker`) - common suite plus Docker-only coverage such as Dockerfile image builds, Docker preflight checks, and managed Docker gateway start. - Docker GPU suite (`--features e2e-docker-gpu`) - Docker suite plus GPU sandbox smoke coverage. - VM suite (`--features e2e-vm`) - runs e2e tests on a VM. @@ -169,6 +172,16 @@ Run the Docker-backed Rust CLI e2e suite: mise run e2e:rust ``` +Run the minimal portable CLI conformance profile against a Docker-backed +gateway: + +```shell +mise run e2e:cli-conformance +``` + +The conformance task selects existing test targets explicitly. Adding a test +to the profile does not remove it from the regular E2E suites. + Run the Podman-backed Rust CLI e2e suite: ```shell diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 0b8b56f3e5..0615846683 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -17,6 +17,10 @@ publish = false [features] e2e = [] +# Selects the portable CLI conformance profile. The profile initially reuses +# existing E2E test targets; membership is defined by the e2e:cli-conformance +# task so adding the feature does not remove tests from existing E2E suites. +e2e-cli-conformance = ["e2e"] # Selects tests that rely on `host.openshell.internal` (the sandbox's stable # alias to the host running test fixtures). docker, podman, and vm wire the # alias unconditionally; the kube driver only does so when the chart's diff --git a/tasks/test.toml b/tasks/test.toml index 9f47d0dcda..6b3f7e0588 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -86,6 +86,12 @@ run = [ "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker", ] +["e2e:cli-conformance"] +description = "Run the portable CLI conformance baseline against a Docker-backed gateway" +run = [ + "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker,e2e-cli-conformance --test smoke", +] + ["e2e:websocket-conformance"] description = "Run focused WebSocket conformance e2e tests against a Docker-backed gateway" run = [ From f90a4742d7bc0c4a0ab3aecf7b9795316ae6797e Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 21 Aug 2026 13:23:32 +0200 Subject: [PATCH 2/4] test(e2e): classify portable CLI workflows Signed-off-by: Evan Lezar --- TESTING.md | 9 +++++---- e2e/rust/Cargo.toml | 15 +++++++++++++++ e2e/rust/e2e-docker.sh | 2 +- e2e/rust/e2e-kubernetes.sh | 2 +- e2e/rust/e2e-podman.sh | 2 +- e2e/rust/e2e-vm.sh | 2 +- tasks/test.toml | 4 ++-- 7 files changed, 26 insertions(+), 10 deletions(-) diff --git a/TESTING.md b/TESTING.md index f20d1b93d9..5eb731c673 100644 --- a/TESTING.md +++ b/TESTING.md @@ -149,8 +149,8 @@ Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. - CLI conformance (`--features e2e-cli-conformance`) - a curated portable CLI - baseline selected from existing E2E test targets. The initial profile contains - the gateway smoke workflow and remains additive to the regular E2E suites. + baseline selected from existing E2E test targets. The profile contains gateway + smoke, sandbox lifecycle, and sandbox label workflows. - Docker suite (`--features e2e-docker`) - common suite plus Docker-only coverage such as Dockerfile image builds, Docker preflight checks, and managed Docker gateway start. - Docker GPU suite (`--features e2e-docker-gpu`) - Docker suite plus GPU sandbox smoke coverage. - VM suite (`--features e2e-vm`) - runs e2e tests on a VM. @@ -179,8 +179,9 @@ gateway: mise run e2e:cli-conformance ``` -The conformance task selects existing test targets explicitly. Adding a test -to the profile does not remove it from the regular E2E suites. +The conformance task selects existing test targets explicitly. Driver suites +enable the same profile when they run overlapping portable targets; the test +implementations are shared instead of duplicated as driver-specific coverage. Run the Podman-backed Rust CLI e2e suite: diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 0615846683..8a9c8d8468 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -45,6 +45,21 @@ name = "oidc_pkce" path = "tests/oidc_pkce.rs" required-features = ["e2e-oidc-pkce"] +[[test]] +name = "smoke" +path = "tests/smoke.rs" +required-features = ["e2e-cli-conformance"] + +[[test]] +name = "sandbox_lifecycle" +path = "tests/sandbox_lifecycle.rs" +required-features = ["e2e-cli-conformance"] + +[[test]] +name = "sandbox_labels" +path = "tests/sandbox_labels.rs" +required-features = ["e2e-cli-conformance"] + [[test]] name = "custom_image" path = "tests/custom_image.rs" diff --git a/e2e/rust/e2e-docker.sh b/e2e/rust/e2e-docker.sh index 6c28868083..c58d1faf00 100755 --- a/e2e/rust/e2e-docker.sh +++ b/e2e/rust/e2e-docker.sh @@ -10,7 +10,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" E2E_TEST="${OPENSHELL_E2E_DOCKER_TEST:-smoke}" -E2E_FEATURES="${OPENSHELL_E2E_DOCKER_FEATURES:-e2e,e2e-docker}" +E2E_FEATURES="${OPENSHELL_E2E_DOCKER_FEATURES:-e2e-docker,e2e-cli-conformance}" DEFAULT_WORKLOAD_MANIFEST="${ROOT}/e2e/gpu/images/.build/workloads.yaml" if [ "${E2E_TEST}" = "gpu" ] && [ -z "${OPENSHELL_E2E_WORKLOAD_MANIFEST:-}" ] && [ ! -f "${DEFAULT_WORKLOAD_MANIFEST}" ]; then diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index cf28e35728..8c148271cd 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -19,7 +19,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -E2E_FEATURES="${OPENSHELL_E2E_KUBERNETES_FEATURES:-e2e,e2e-host-gateway,e2e-kubernetes}" +E2E_FEATURES="${OPENSHELL_E2E_KUBERNETES_FEATURES:-e2e,e2e-cli-conformance,e2e-host-gateway,e2e-kubernetes}" # Docker and Podman build their local gateway and CLI together in the shared # gateway wrapper. Kubernetes consumes published gateway images, so only its diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index 16796c0562..8cadb409dc 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -10,7 +10,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" E2E_TEST="${OPENSHELL_E2E_PODMAN_TEST:-}" -E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES:-e2e-podman}" +E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES:-e2e-podman,e2e-cli-conformance}" DEFAULT_WORKLOAD_MANIFEST="${ROOT}/e2e/gpu/images/.build/workloads.yaml" if [ "${E2E_TEST}" = "gpu" ] && [ -z "${OPENSHELL_E2E_WORKLOAD_MANIFEST:-}" ] && [ ! -f "${DEFAULT_WORKLOAD_MANIFEST}" ]; then diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 1960f83588..f6bb4bd664 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -52,7 +52,7 @@ GATEWAY_BIN="${OPENSHELL_GATEWAY_BIN:-${ROOT}/target/debug/openshell-gateway}" DRIVER_BIN="${OPENSHELL_VM_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-vm}" CLI_BIN="${OPENSHELL_BIN:-${ROOT}/target/debug/openshell}" E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}" -E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm}" +E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm,e2e-cli-conformance}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}}" # The VM driver places `compute-driver.sock` under `[openshell.drivers.vm].state_dir`. diff --git a/tasks/test.toml b/tasks/test.toml index 6b3f7e0588..ea77f23295 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -83,13 +83,13 @@ hide = true ["e2e:rust"] description = "Run Rust CLI e2e tests against a Docker-backed gateway" run = [ - "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker", + "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker,e2e-cli-conformance", ] ["e2e:cli-conformance"] description = "Run the portable CLI conformance baseline against a Docker-backed gateway" run = [ - "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker,e2e-cli-conformance --test smoke", + "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker,e2e-cli-conformance --test smoke --test sandbox_lifecycle --test sandbox_labels", ] ["e2e:websocket-conformance"] From af52d89c882909aa1033c45b1a99e8d040e2ee3c Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 21 Aug 2026 13:25:13 +0200 Subject: [PATCH 3/4] test(e2e): isolate VM-specific smoke assertions Signed-off-by: Evan Lezar --- TESTING.md | 2 ++ e2e/rust/Cargo.toml | 5 ++++ e2e/rust/e2e-vm.sh | 2 +- e2e/rust/tests/smoke.rs | 42 ---------------------------- e2e/rust/tests/vm_overlay.rs | 54 ++++++++++++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 43 deletions(-) create mode 100644 e2e/rust/tests/vm_overlay.rs diff --git a/TESTING.md b/TESTING.md index 5eb731c673..5e7b72ddff 100644 --- a/TESTING.md +++ b/TESTING.md @@ -182,6 +182,8 @@ mise run e2e:cli-conformance The conformance task selects existing test targets explicitly. Driver suites enable the same profile when they run overlapping portable targets; the test implementations are shared instead of duplicated as driver-specific coverage. +VM overlay and TLS-key permission assertions remain in the VM suite instead of +the portable smoke workflow. Run the Podman-backed Rust CLI e2e suite: diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 8a9c8d8468..19702aadf1 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -60,6 +60,11 @@ name = "sandbox_labels" path = "tests/sandbox_labels.rs" required-features = ["e2e-cli-conformance"] +[[test]] +name = "vm_overlay" +path = "tests/vm_overlay.rs" +required-features = ["e2e-vm"] + [[test]] name = "custom_image" path = "tests/custom_image.rs" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index f6bb4bd664..7eca7ddb37 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -371,7 +371,6 @@ fi # The CLI uses the raw endpoint but still resolves matching metadata so it # can find the mTLS client bundle. -export OPENSHELL_E2E_EXPECT_VM_OVERLAY=1 export OPENSHELL_E2E_DRIVER="vm" export OPENSHELL_E2E_VM_STATE_DIR="${RUN_STATE_DIR}" e2e_export_gateway_restart_metadata \ @@ -405,5 +404,6 @@ if [ -n "${E2E_TEST_OVERRIDE}" ]; then else run_e2e_test smoke run_e2e_test host_gateway_alias + run_e2e_test vm_overlay run_e2e_test vm_gateway_start fi diff --git a/e2e/rust/tests/smoke.rs b/e2e/rust/tests/smoke.rs index c27255e5ef..172afa22b8 100644 --- a/e2e/rust/tests/smoke.rs +++ b/e2e/rust/tests/smoke.rs @@ -68,10 +68,6 @@ async fn gateway_smoke() { sb.create_output, ); - if std::env::var_os("OPENSHELL_E2E_EXPECT_VM_OVERLAY").is_some() { - assert_vm_overlay_root(&sb.name).await; - } - // ── 3. Verify the sandbox appeared in the list ─────────────────── let mut list_cmd = openshell_cmd(); list_cmd @@ -99,41 +95,3 @@ async fn gateway_smoke() { // ── 4. Cleanup ─────────────────────────────────────────────────── sb.cleanup().await; } - -async fn assert_vm_overlay_root(sandbox_name: &str) { - let script = concat!( - "set -eu; ", - "test \"$(stat -f -c %T /)\" = \"overlayfs\"; ", - "printf \"overlay-write\\n\" > /sandbox/overlay-check; ", - "test \"$(cat /sandbox/overlay-check)\" = \"overlay-write\"; ", - "if [ -e /opt/openshell/tls/tls.key ]; then ", - "test \"$(stat -c %a /opt/openshell/tls/tls.key)\" = \"600\"; ", - "fi; ", - "echo vm-overlay-ok", - ); - - let mut exec_cmd = openshell_cmd(); - exec_cmd - .args(["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"]) - .arg("sh") - .arg("-lc") - .arg(script) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = exec_cmd - .output() - .await - .expect("failed to run VM overlay assertion"); - let combined = strip_ansi(&format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - - assert!( - output.status.success() && combined.contains("vm-overlay-ok"), - "VM overlay assertion failed (status {:?}):\n{combined}", - output.status.code(), - ); -} diff --git a/e2e/rust/tests/vm_overlay.rs b/e2e/rust/tests/vm_overlay.rs new file mode 100644 index 0000000000..078c6f01de --- /dev/null +++ b/e2e/rust/tests/vm_overlay.rs @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! VM-driver-specific assertions for the sandbox root filesystem. + +use std::process::Stdio; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::output::strip_ansi; +use openshell_e2e::harness::sandbox::SandboxGuard; + +#[tokio::test] +async fn vm_overlay() { + let mut sandbox = SandboxGuard::create(&["--", "echo", "vm-sandbox-ready"]) + .await + .expect("sandbox create should succeed"); + + let script = concat!( + "set -eu; ", + "test \"$(stat -f -c %T /)\" = \"overlayfs\"; ", + "printf \"overlay-write\\n\" > /sandbox/overlay-check; ", + "test \"$(cat /sandbox/overlay-check)\" = \"overlay-write\"; ", + "if [ -e /opt/openshell/tls/tls.key ]; then ", + "test \"$(stat -c %a /opt/openshell/tls/tls.key)\" = \"600\"; ", + "fi; ", + "echo vm-overlay-ok", + ); + + let mut exec_cmd = openshell_cmd(); + exec_cmd + .args(["sandbox", "exec", "--name", &sandbox.name, "--no-tty", "--"]) + .arg("sh") + .arg("-lc") + .arg(script) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let output = exec_cmd + .output() + .await + .expect("failed to run VM overlay assertion"); + let combined = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )); + assert!( + output.status.success() && combined.contains("vm-overlay-ok"), + "VM overlay assertion failed (status {:?}):\n{combined}", + output.status.code(), + ); + + sandbox.cleanup().await; +} From 991cfce4c0c033434cf13ec760575cc536f673f7 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 24 Aug 2026 15:39:13 +0200 Subject: [PATCH 4/4] test(e2e): decouple CLI conformance provisioning Signed-off-by: Evan Lezar --- TESTING.md | 17 +++++++++++------ tasks/test.toml | 5 +++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/TESTING.md b/TESTING.md index 5e7b72ddff..e3bbd877e6 100644 --- a/TESTING.md +++ b/TESTING.md @@ -172,18 +172,23 @@ Run the Docker-backed Rust CLI e2e suite: mise run e2e:rust ``` -Run the minimal portable CLI conformance profile against a Docker-backed -gateway: +Run the minimal portable CLI conformance profile against the gateway selected +in your OpenShell CLI configuration: ```shell mise run e2e:cli-conformance ``` +The gateway must already be installed, reachable, and selected before the task +starts. The task does not provision a gateway or select a compute driver. Set +`OPENSHELL_BIN` to test a prebuilt CLI; otherwise, the task builds the CLI from +the current checkout. + The conformance task selects existing test targets explicitly. Driver suites -enable the same profile when they run overlapping portable targets; the test -implementations are shared instead of duplicated as driver-specific coverage. -VM overlay and TLS-key permission assertions remain in the VM suite instead of -the portable smoke workflow. +provision their own gateways and enable the same profile when they run +overlapping portable targets; the test implementations are shared instead of +duplicated as driver-specific coverage. VM overlay and TLS-key permission +assertions remain in the VM suite instead of the portable smoke workflow. Run the Podman-backed Rust CLI e2e suite: diff --git a/tasks/test.toml b/tasks/test.toml index ea77f23295..9c8d223fb8 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -87,9 +87,10 @@ run = [ ] ["e2e:cli-conformance"] -description = "Run the portable CLI conformance baseline against a Docker-backed gateway" +description = "Run the portable CLI conformance baseline against the configured gateway" run = [ - "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker,e2e-cli-conformance --test smoke --test sandbox_lifecycle --test sandbox_labels", + "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli; fi", + "cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-cli-conformance --test smoke --test sandbox_lifecycle --test sandbox_labels", ] ["e2e:websocket-conformance"]