From 05085d010d8165ec6a4b4f5dfbe13769bf4c9ce8 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Thu, 2 Jul 2026 14:14:07 -0700 Subject: [PATCH 01/25] feat(kubernetes): add proxy-pod supervisor topology Add the Kubernetes proxy-pod topology with one supervisor Deployment and Service per sandbox, NetworkPolicy confinement, proxy-pod Helm/Skaffold configuration, topology documentation, and focused supervisor identity tests. Signed-off-by: Taylor Mutch --- .../skills/debug-openshell-cluster/SKILL.md | 23 +- .agents/skills/helm-dev-environment/SKILL.md | 52 +- Cargo.lock | 4 + Cargo.toml | 2 +- architecture/gateway.md | 8 +- crates/openshell-core/src/sandbox_env.rs | 23 + crates/openshell-driver-kubernetes/Cargo.toml | 1 + crates/openshell-driver-kubernetes/README.md | 8 + .../openshell-driver-kubernetes/src/config.rs | 73 +- .../openshell-driver-kubernetes/src/driver.rs | 1625 +++++++++++++++-- crates/openshell-driver-kubernetes/src/lib.rs | 6 +- .../openshell-driver-kubernetes/src/main.rs | 15 +- crates/openshell-sandbox/src/lib.rs | 171 +- crates/openshell-sandbox/src/main.rs | 9 +- crates/openshell-server/src/auth/k8s_sa.rs | 290 ++- .../src/l7/tls.rs | 36 + .../openshell-supervisor-network/src/run.rs | 50 +- .../openshell-supervisor-process/Cargo.toml | 1 + .../src/netns/mod.rs | 2 +- .../src/process.rs | 73 +- .../openshell-supervisor-process/src/run.rs | 59 +- deploy/helm/openshell/README.md | 3 +- .../helm/openshell/ci/values-proxy-pod.yaml | 18 + deploy/helm/openshell/skaffold.yaml | 10 + .../openshell/templates/gateway-config.yaml | 3 + deploy/helm/openshell/templates/role.yaml | 50 +- .../openshell/tests/gateway_config_test.yaml | 21 + .../tests/sandbox_namespace_test.yaml | 133 ++ deploy/helm/openshell/values.yaml | 6 + docs/kubernetes/setup.mdx | 7 +- docs/kubernetes/topology.mdx | 120 +- docs/reference/gateway-config.mdx | 6 + docs/reference/sandbox-compute-drivers.mdx | 11 +- e2e/rust/tests/live_policy_update.rs | 13 +- e2e/with-kube-gateway.sh | 15 + tasks/helm.toml | 15 + tasks/scripts/helm-k3s-local.sh | 4 + tasks/test.toml | 5 + 38 files changed, 2692 insertions(+), 279 deletions(-) create mode 100644 deploy/helm/openshell/ci/values-proxy-pod.yaml diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 8e51a73932..9a34e0f893 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -505,7 +505,28 @@ The shared state directory should preserve `sandbox_gid` inheritance `@openshell-sidecar-ssh`; the network sidecar verifies its peer PID before bridging gateway relay requests. No `ssh.sock` file should appear in the shared state directory. -Inspect all three when sandbox registration or egress enforcement fails: + +If `topology = "proxy-pod"` is rendered, each sandbox should have a +separate supervisor Deployment with one supervisor pod, a headless supervisor +Service, a proxy CA Secret, and two per-sandbox NetworkPolicies. The agent pod +should have `openshell.ai/sandbox-role=agent`; the supervisor pod should have +`openshell.ai/sandbox-role=supervisor`; both should share the same +`openshell.ai/sandbox-id`. The supervisor Deployment must have a controlling +`Sandbox` ownerReference. The Deployment pod template must carry the +`openshell.io/sandbox-id` annotation so the TokenReview bootstrap path can mint +a sandbox JWT. For supervisor pods, the gateway validates the +`Pod -> ReplicaSet -> Deployment -> Sandbox` owner chain, so missing +`apps/replicasets get` RBAC can also break bootstrap. Helm renders the +Deployment, ReplicaSet, Service, Secret, and NetworkPolicy RBAC only when +`supervisor.topology=proxy-pod`; if those resources fail with forbidden errors, +confirm both the rendered `gateway.toml` and Helm values use proxy-pod topology. +If the agent cannot reach the gateway, check DNS to the headless Service, the +agent egress NetworkPolicy DNS exception for kube-dns/CoreDNS, and the +supervisor ingress NetworkPolicy allowing only that agent pod on ports `3128` +and `18080`. + +Inspect the relevant containers when sandbox registration or egress enforcement +fails: ```bash kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\.toml}' | grep -E '^\[openshell\.drivers\.kubernetes\]|^topology\s*=' diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 2dad568c79..37f738ad6c 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -70,16 +70,28 @@ mise run helm:skaffold:run:sidecar mise run helm:skaffold:run:sidecar-mtls ``` -Both commands build the `gateway` and `supervisor` images and deploy the OpenShell Helm +**Supervisor proxy-pod topology** (build once and leave running): +```bash +mise run helm:skaffold:run:proxy-pod +``` + +All Skaffold commands build the `gateway` and `supervisor` images and deploy the OpenShell Helm chart. The sidecar profile renders an `openshell-network-init` init container for nftables setup and an `openshell-supervisor-network` runtime sidecar for proxying. Binary-aware policy mode runs that sidecar as UID 0 with `SYS_PTRACE` and `DAC_READ_SEARCH`; relaxed mode can run it as the configured proxy UID, which must be at least `1000` and distinct from the workload UID. The sidecar-mTLS profile reuses `ci/values-sidecar.yaml` and restores -`server.disableTls=false` inline for Skaffold. The `pkiInitJob` hook (a pre-install -Job that runs `openshell-gateway generate-certs`) generates mTLS secrets on first -install. Envoy Gateway opt-in; see the Optional Add-ons section below. +`server.disableTls=false` inline for Skaffold. The proxy-pod profile renders +network supervision in a separate supervisor Deployment with one pod and relies +on Kubernetes NetworkPolicy enforcement so the agent pod can reach only its +paired supervisor plus DNS. The +default local k3s/k3d cluster keeps k3s's embedded NetworkPolicy controller +enabled; if you replace the CNI, install a policy-enforcing CNI before using +proxy-pod. The +`pkiInitJob` hook (a pre-install Job that runs `openshell-gateway +generate-certs`) generates mTLS secrets on first install. Envoy Gateway opt-in; +see the Optional Add-ons section below. The gateway Service uses ClusterIP. Access is via Envoy Gateway (port `8080`) or `kubectl port-forward`. @@ -88,6 +100,31 @@ The gateway Service uses ClusterIP. Access is via Envoy Gateway (port `8080`) or create the Secret named `openshell-ha-pg` with a `uri` key, then run `mise run helm:skaffold:run` or `mise run helm:skaffold:dev`. +### Kubernetes e2e profiles + +Run the default Kubernetes e2e environment: + +```bash +mise run e2e:kubernetes +``` + +Run the sidecar topology e2e environment: + +```bash +mise run e2e:kubernetes:sidecar +``` + +Run the proxy-pod topology e2e environment: + +```bash +mise run e2e:kubernetes:proxy-pod +``` + +The proxy-pod e2e task applies `ci/values-proxy-pod.yaml` through +`OPENSHELL_E2E_KUBE_EXTRA_VALUES`. Use an existing cluster with NetworkPolicy +enforcement, or let the wrapper create the default local k3d/k3s cluster with +k3s's embedded NetworkPolicy controller enabled. + ### TLS behaviour `ci/values-skaffold.yaml` sets `server.disableTls: true`, so Skaffold-based deploys run @@ -150,6 +187,12 @@ For a sidecar-profile deployment: mise run helm:skaffold:delete:sidecar ``` +For a proxy-pod-profile deployment: + +```bash +mise run helm:skaffold:delete:proxy-pod +``` + ### Delete the cluster entirely ```bash @@ -275,6 +318,7 @@ for dependencies still declared in `Chart.yaml`. | `deploy/helm/openshell/ci/values-high-availability.yaml` | HA test overlay (`replicaCount: 2` with external PostgreSQL Secret) | | `deploy/helm/openshell/ci/values-keycloak.yaml` | Keycloak OIDC overlay | | `deploy/helm/openshell/ci/values-sidecar.yaml` | Supervisor sidecar topology overlay for Kubernetes e2e/dev | +| `deploy/helm/openshell/ci/values-proxy-pod.yaml` | Supervisor proxy-pod topology overlay for Kubernetes e2e/dev; requires NetworkPolicy enforcement | | `deploy/helm/openshell/ci/values-spire.yaml` | SPIFFE/SPIRE provider token grant overlay | | `deploy/helm/openshell/ci/values-spire-stack.yaml` | SPIRE hardened chart values for local dev | | `deploy/helm/openshell/ci/values-tls-disabled.yaml` | Lint-only: TLS + auth disabled (reverse-proxy edge termination) | diff --git a/Cargo.lock b/Cargo.lock index a9eee33af9..0d6e5c8279 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3849,6 +3849,7 @@ dependencies = [ "openshell-policy", "prost", "prost-types", + "rcgen", "serde", "serde_json", "temp-env", @@ -4367,6 +4368,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "socket2 0.6.3", + "temp-env", "tempfile", "tokio", "tokio-stream", @@ -5341,6 +5343,7 @@ dependencies = [ "ring", "rustls-pki-types", "time", + "x509-parser", "yasna", ] @@ -8454,6 +8457,7 @@ dependencies = [ "lazy_static", "nom", "oid-registry", + "ring", "rusticata-macros", "thiserror 1.0.69", "time", diff --git a/Cargo.toml b/Cargo.toml index c484ec95b1..271df97a17 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ http-body-util = "0.1" tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "ring"] } rustls = { version = "0.23", default-features = false, features = ["std", "logging", "tls12", "ring"] } rustls-pemfile = "2" -rcgen = { version = "0.13", features = ["crypto", "pem"] } +rcgen = { version = "0.13", features = ["crypto", "pem", "x509-parser"] } webpki-roots = "1" rustls-native-certs = "0.8" diff --git a/architecture/gateway.md b/architecture/gateway.md index 32bca6a1f6..829c4e13c8 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -219,9 +219,11 @@ Podman, and VM drivers deliver the initial token through supervisor-only runtime material; Kubernetes supervisors exchange a projected ServiceAccount token through `IssueSandboxToken`. The gateway validates that projected token with Kubernetes `TokenReview`, requires the configured sandbox service account, -checks the returned pod binding against the live pod UID, and verifies the pod's -controlling `Sandbox` ownerReference against the live Sandbox CR UID and -sandbox-id label before minting the gateway JWT. The bootstrap path accepts +checks the returned pod binding against the live pod UID, and verifies the +pod's ownership against the live Sandbox CR UID and sandbox-id label before +minting the gateway JWT. Agent pods must be directly controlled by the +`Sandbox` CR. Proxy-pod supervisor pods may be controlled through the Kubernetes +`Pod -> ReplicaSet -> Deployment -> Sandbox` chain. The bootstrap path accepts both `agents.x-k8s.io/v1beta1` ownerReferences from newer Agent Sandbox controllers and `agents.x-k8s.io/v1alpha1` ownerReferences from existing deployments. Supervisors renew gateway JWTs in memory before expiry only while diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 40a7f0a72f..81e6953a43 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -143,13 +143,36 @@ pub const NETWORK_BINARY_IDENTITY: &str = "OPENSHELL_NETWORK_BINARY_IDENTITY"; /// container. pub const SIDECAR_CONTROL_SOCKET: &str = "OPENSHELL_SIDECAR_CONTROL_SOCKET"; +/// TCP address the process supervisor waits for before starting when the +/// network supervisor runs outside the agent process. +pub const SUPERVISOR_READY_ADDR: &str = "OPENSHELL_SUPERVISOR_READY_ADDR"; + +/// Address where an external network supervisor forwards gateway gRPC traffic. +pub const GATEWAY_FORWARD_ADDR: &str = "OPENSHELL_GATEWAY_FORWARD_ADDR"; + /// Optional TLS server name override used when connecting to the gateway. pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME"; +/// Explicit URL injected into sandbox child processes for proxy-mode egress. +/// +/// Kubernetes proxy-pod topology uses a headless Service DNS name, which +/// cannot be represented by the policy's `SocketAddr` proxy field. +pub const PROXY_URL: &str = "OPENSHELL_PROXY_URL"; + +/// Explicit listener address for the network supervisor's HTTP CONNECT proxy. +pub const PROXY_BIND_ADDR: &str = "OPENSHELL_PROXY_BIND_ADDR"; + /// Directory where the network supervisor writes the proxy CA files consumed /// by workload child processes. pub const PROXY_TLS_DIR: &str = "OPENSHELL_PROXY_TLS_DIR"; +/// Optional CA certificate PEM path used by the network supervisor instead of +/// generating an ephemeral CA. +pub const PROXY_CA_CERT_PATH: &str = "OPENSHELL_PROXY_CA_CERT_PATH"; + +/// Optional CA private key PEM path paired with [`PROXY_CA_CERT_PATH`]. +pub const PROXY_CA_KEY_PATH: &str = "OPENSHELL_PROXY_CA_KEY_PATH"; + /// Path to the CA certificate for mTLS communication with the gateway. pub const TLS_CA: &str = "OPENSHELL_TLS_CA"; diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 714b7d05c9..5374007712 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -35,6 +35,7 @@ tracing-subscriber = { workspace = true } thiserror = { workspace = true } miette = { workspace = true } notify = "8" +rcgen = { workspace = true } [dev-dependencies] temp-env = "0.3" diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index fac220c83b..26c4413e81 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -135,6 +135,14 @@ abstract socket whose peer PID must match that authenticated supervisor. Both supervisors exit if the control connection closes, coupling their container restart lifecycle before a new authoritative client can be established. +The `proxy-pod` supervisor topology runs network enforcement and gateway +forwarding in a separate supervisor Deployment with one pod. The agent pod runs +only the process-mode supervisor and reaches the supervisor through a +per-sandbox headless Service. The driver creates an owner-referenced supervisor +Deployment with one replica plus Service, proxy CA Secret, and NetworkPolicy +resources so agent egress is limited to its paired supervisor pod plus DNS. If +the supervisor pod is deleted, the Deployment recreates it. + The driver can request a Kubernetes AppArmor profile through `app_armor_profile`. diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index aedd3b8bff..5284ee0131 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -22,7 +22,7 @@ pub const DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME: &str = "default"; /// Default storage size for the workspace PVC. pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi"; -/// Default non-root UID for relaxed Kubernetes network supervisor sidecars. +/// Default UID for the long-running Kubernetes network proxy. pub const DEFAULT_PROXY_UID: u32 = 1337; /// How the supervisor binary is delivered into sandbox pods. @@ -72,6 +72,9 @@ pub enum SupervisorTopology { /// Run network supervision in a privileged sidecar and process supervision /// as a low-capability wrapper in the agent container. Sidecar, + /// Run network supervision in a separate supervisor pod and process + /// supervision as a low-capability wrapper in the agent pod. + ProxyPod, } impl std::fmt::Display for SupervisorTopology { @@ -79,6 +82,7 @@ impl std::fmt::Display for SupervisorTopology { match self { Self::Combined => f.write_str("combined"), Self::Sidecar => f.write_str("sidecar"), + Self::ProxyPod => f.write_str("proxy-pod"), } } } @@ -90,6 +94,7 @@ impl FromStr for SupervisorTopology { match s { "combined" => Ok(Self::Combined), "sidecar" => Ok(Self::Sidecar), + "proxy-pod" => Ok(Self::ProxyPod), other => Err(format!("unknown topology '{other}'")), } } @@ -177,6 +182,34 @@ impl KubernetesSidecarConfig { } } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct KubernetesProxyPodConfig { + /// UID used by the network supervisor in `proxy-pod` topology. It must not + /// match the sandbox workload UID. + pub proxy_uid: u32, +} + +impl Default for KubernetesProxyPodConfig { + fn default() -> Self { + Self { + proxy_uid: DEFAULT_PROXY_UID, + } + } +} + +impl KubernetesProxyPodConfig { + pub fn validate_proxy_uid(&self) -> Result<(), String> { + if self.proxy_uid < openshell_policy::MIN_SANDBOX_UID { + return Err(format!( + "proxy_pod.proxy_uid must be at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + Ok(()) + } +} + /// Kubernetes `AppArmor` profile requested for the sandbox agent container. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AppArmorProfile { @@ -326,6 +359,8 @@ pub struct KubernetesComputeConfig { pub topology: SupervisorTopology, /// Sidecar-only settings used when `topology = "sidecar"`. pub sidecar: KubernetesSidecarConfig, + /// Proxy-pod-only settings used when `topology = "proxy-pod"`. + pub proxy_pod: KubernetesProxyPodConfig, /// Corporate HTTP forward proxy used by the network supervisor for /// policy-approved TLS CONNECT egress. pub https_proxy: Option, @@ -451,6 +486,7 @@ impl Default for KubernetesComputeConfig { supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), sidecar: KubernetesSidecarConfig::default(), + proxy_pod: KubernetesProxyPodConfig::default(), https_proxy: None, no_proxy: None, proxy_auth_secret_name: None, @@ -503,7 +539,8 @@ impl KubernetesComputeConfig { } pub fn validate_proxy_uid(&self) -> Result<(), String> { - self.sidecar.validate_proxy_uid() + self.sidecar.validate_proxy_uid()?; + self.proxy_pod.validate_proxy_uid() } /// Validate the operator-owned corporate upstream proxy configuration. @@ -578,11 +615,11 @@ impl KubernetesComputeConfig { if self.proxy_auth_allow_insecure != Some(true) { return Err("proxy credentials use cleartext Basic auth over the connection to the http:// proxy; set proxy_auth_allow_insecure = true to accept that exposure, or remove the credential Secret".to_string()); } - if self.topology == SupervisorTopology::Combined { - return Err( - "proxy credential Secrets require topology = \"sidecar\"; combined topology shares the credential mount with the workload and fsGroup can make it readable by the sandbox user" - .to_string(), - ); + if self.topology != SupervisorTopology::Sidecar { + return Err(format!( + "proxy credential Secrets require topology = \"sidecar\"; {} topology does not mount the credential into a supervisor container isolated from the workload", + self.topology + )); } } _ => { @@ -946,6 +983,28 @@ mod tests { assert_eq!(cfg.topology, SupervisorTopology::Combined); } + #[test] + fn serde_override_topology_proxy_pod() { + let json = serde_json::json!({ + "topology": "proxy-pod" + }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.topology, SupervisorTopology::ProxyPod); + assert_eq!(cfg.topology.to_string(), "proxy-pod"); + } + + #[test] + fn serde_override_proxy_pod_proxy_uid_nested() { + let json = serde_json::json!({ + "proxy_pod": { + "proxy_uid": 2000 + } + }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.proxy_pod.proxy_uid, 2000); + cfg.validate_proxy_uid().unwrap(); + } + #[test] fn serde_rejects_sidecar_binary_identity_field() { let json = serde_json::json!({ diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 84d7029de4..1d607e1902 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -11,9 +11,10 @@ use crate::config::{ managed_namespace, validate_managed_namespace_name, }; use futures::{Stream, StreamExt, TryStreamExt}; +use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::api::core::v1::{ Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Pod, Secret, - ServiceAccount, Volume, VolumeMount, + Service, ServiceAccount, Volume, VolumeMount, }; use k8s_openapi::api::networking::v1::{ NetworkPolicy, NetworkPolicyIngressRule, NetworkPolicyPeer, NetworkPolicyPort, @@ -48,7 +49,9 @@ use openshell_core::proto::compute::v1::{ watch_sandboxes_event, }; use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; +use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose}; use serde::Deserialize; +use serde::de::DeserializeOwned; use std::collections::{BTreeMap, HashSet}; use std::path::{Path, PathBuf}; use std::pin::Pin; @@ -1406,7 +1409,12 @@ impl KubernetesComputeDriver { supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, supervisor_sideload_method: self.config.supervisor_sideload_method, topology: self.config.topology, - proxy_uid: self.config.sidecar.proxy_uid, + proxy_uid: match self.config.topology { + SupervisorTopology::ProxyPod => self.config.proxy_pod.proxy_uid, + SupervisorTopology::Combined | SupervisorTopology::Sidecar => { + self.config.sidecar.proxy_uid + } + }, process_binary_aware_network_policy: self .config .sidecar @@ -1417,6 +1425,7 @@ impl KubernetesComputeDriver { proxy_auth_secret_key: self.config.proxy_auth_secret_key.as_deref(), proxy_auth_allow_insecure: self.config.proxy_auth_allow_insecure == Some(true), proxy_connect_by_hostname: self.config.proxy_connect_by_hostname == Some(true), + namespace: &self.config.namespace, service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, sandbox_name: &sandbox.name, @@ -1437,7 +1446,7 @@ impl KubernetesComputeDriver { sandbox_uid: resolved_user_id, sandbox_gid: resolved_group_id, }; - validate_sidecar_proxy_identity(¶ms)?; + validate_proxy_identity(¶ms)?; let data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) .map_err(KubernetesDriverError::InvalidArgument)?; @@ -1461,19 +1470,19 @@ impl KubernetesComputeDriver { }; obj.data = data; - match tokio::time::timeout( + let created = match tokio::time::timeout( KUBE_API_TIMEOUT, agent_sandbox_api.api.create(&PostParams::default(), &obj), ) .await { - Ok(Ok(_result)) => { + Ok(Ok(result)) => { info!( sandbox_id = %sandbox.id, sandbox_name = %name, "Sandbox created in Kubernetes successfully" ); - Ok(()) + result } Ok(Err(err)) => { warn!( @@ -1482,7 +1491,7 @@ impl KubernetesComputeDriver { error = %err, "Failed to create sandbox in Kubernetes" ); - Err(KubernetesDriverError::from_kube(err)) + return Err(KubernetesDriverError::from_kube(err)); } Err(_elapsed) => { warn!( @@ -1491,12 +1500,196 @@ impl KubernetesComputeDriver { timeout_secs = KUBE_API_TIMEOUT.as_secs(), "Timed out creating sandbox in Kubernetes" ); - Err(KubernetesDriverError::Message(format!( + return Err(KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes API", KUBE_API_TIMEOUT.as_secs() - ))) + ))); } + }; + + if self.config.topology == SupervisorTopology::ProxyPod + && let Err(err) = self + .create_proxy_pod_resources( + sandbox, + sandbox.spec.as_ref(), + ¶ms, + &created, + &agent_sandbox_api.resource.api_version, + ) + .await + { + warn!( + sandbox_id = %sandbox.id, + sandbox_name = %name, + error = %err, + "Failed to create proxy-pod resources; deleting Sandbox CR" + ); + self.cleanup_proxy_pod_resources(name, &self.config.namespace) + .await; + let _ = tokio::time::timeout( + KUBE_API_TIMEOUT, + agent_sandbox_api.api.delete(name, &DeleteParams::default()), + ) + .await; + return Err(err); } + + Ok(()) + } + + async fn create_proxy_pod_resources( + &self, + sandbox: &Sandbox, + spec: Option<&SandboxSpec>, + params: &SandboxPodParams<'_>, + sandbox_cr: &DynamicObject, + sandbox_api_version: &str, + ) -> Result<(), KubernetesDriverError> { + let names = proxy_pod_resource_names(&sandbox.name); + let template_environment = spec + .and_then(|spec| spec.template.as_ref()) + .map(|template| template.environment.clone()) + .unwrap_or_default(); + let spec_environment = spec_pod_env(spec); + let deployment_owner_ref = + proxy_pod_owner_reference(sandbox_cr, sandbox_api_version, true)?; + let dependent_owner_ref = + proxy_pod_owner_reference(sandbox_cr, sandbox_api_version, false)?; + let (ca_cert_pem, ca_key_pem) = generate_proxy_pod_ca()?; + + let secret = proxy_pod_ca_secret( + &names, + params, + dependent_owner_ref.clone(), + &ca_cert_pem, + &ca_key_pem, + ); + let service = proxy_pod_supervisor_service(&names, params, dependent_owner_ref.clone()); + let agent_egress = + proxy_pod_agent_egress_network_policy(&names, params, dependent_owner_ref.clone()); + let supervisor_ingress = + proxy_pod_supervisor_ingress_network_policy(&names, params, dependent_owner_ref); + let supervisor_deployment = proxy_pod_supervisor_deployment( + &names, + &template_environment, + &spec_environment, + params, + deployment_owner_ref, + ); + + let secrets: Api = Api::namespaced(self.client.clone(), &self.config.namespace); + let services: Api = Api::namespaced(self.client.clone(), &self.config.namespace); + let policies: Api = + Api::namespaced(self.client.clone(), &self.config.namespace); + let deployments: Api = + Api::namespaced(self.client.clone(), &self.config.namespace); + + tokio::time::timeout( + KUBE_API_TIMEOUT, + secrets.create(&PostParams::default(), &secret), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s creating proxy-pod CA secret", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + tokio::time::timeout( + KUBE_API_TIMEOUT, + services.create(&PostParams::default(), &service), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s creating proxy-pod service", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.create(&PostParams::default(), &agent_egress), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s creating proxy-pod agent egress NetworkPolicy", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.create(&PostParams::default(), &supervisor_ingress), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s creating proxy-pod supervisor ingress NetworkPolicy", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + tokio::time::timeout( + KUBE_API_TIMEOUT, + deployments.create(&PostParams::default(), &supervisor_deployment), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s creating proxy-pod supervisor deployment", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + + info!( + sandbox_id = %sandbox.id, + sandbox_name = %sandbox.name, + supervisor_deployment = %names.supervisor_deployment, + service = %names.service, + "Created proxy-pod supervisor resources" + ); + Ok(()) + } + + async fn cleanup_proxy_pod_resources(&self, sandbox_name: &str, namespace: &str) { + let names = proxy_pod_resource_names(sandbox_name); + let secrets: Api = Api::namespaced(self.client.clone(), namespace); + let services: Api = Api::namespaced(self.client.clone(), namespace); + let policies: Api = Api::namespaced(self.client.clone(), namespace); + let deployments: Api = Api::namespaced(self.client.clone(), namespace); + + let _ = tokio::time::timeout( + KUBE_API_TIMEOUT, + deployments.delete(&names.supervisor_deployment, &DeleteParams::default()), + ) + .await; + let _ = tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.delete( + &names.supervisor_ingress_network_policy, + &DeleteParams::default(), + ), + ) + .await; + let _ = tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.delete(&names.agent_egress_network_policy, &DeleteParams::default()), + ) + .await; + let _ = tokio::time::timeout( + KUBE_API_TIMEOUT, + services.delete(&names.service, &DeleteParams::default()), + ) + .await; + let _ = tokio::time::timeout( + KUBE_API_TIMEOUT, + secrets.delete(&names.proxy_ca_secret, &DeleteParams::default()), + ) + .await; } pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { @@ -1715,6 +1908,11 @@ impl KubernetesComputeDriver { } }; + if self.config.topology == SupervisorTopology::ProxyPod { + self.cleanup_proxy_pod_resources(&kube_name, &obj_namespace) + .await; + } + let delete_api = self .supported_agent_sandbox_api(self.client.clone(), &obj_namespace) .await?; @@ -2368,6 +2566,18 @@ const SIDECAR_TLS_VOLUME_NAME: &str = "openshell-supervisor-tls"; const SIDECAR_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; const SIDECAR_CLIENT_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_CLIENT_TLS_DIR; +const LABEL_SANDBOX_ROLE: &str = "openshell.ai/sandbox-role"; +const SANDBOX_ROLE_AGENT: &str = "agent"; +const SANDBOX_ROLE_SUPERVISOR: &str = "supervisor"; +const PROXY_POD_PROXY_PORT: u16 = 3128; +const PROXY_POD_GATEWAY_FORWARD_PORT: u16 = 18080; +const PROXY_POD_GATEWAY_FORWARD_ADDR: &str = "0.0.0.0:18080"; +const PROXY_POD_NETWORK_ENFORCEMENT_MODE: &str = "proxy-pod"; +const PROXY_POD_CA_SECRET_MOUNT_PATH: &str = "/var/run/openshell-proxy-ca"; +const PROXY_POD_CA_CERT_FILE: &str = "openshell-ca.pem"; +const PROXY_POD_CA_KEY_FILE: &str = "openshell-ca-key.pem"; +const PROXY_POD_SSH_SOCKET_FILE: &str = "/tmp/openshell/ssh.sock"; + /// Build the emptyDir volume that holds the supervisor binary. /// /// The init container writes the binary here; the agent container reads it. @@ -2658,6 +2868,111 @@ fn sidecar_tls_volume_mount() -> serde_json::Value { }) } +fn gateway_tls_server_name(grpc_endpoint: &str) -> Option { + let rest = grpc_endpoint.strip_prefix("https://")?; + let authority = rest.split('/').next().unwrap_or(rest); + if authority.is_empty() { + return None; + } + if let Some(bracketed) = authority.strip_prefix('[') { + return bracketed.split(']').next().map(str::to_string); + } + authority + .split(':') + .next() + .filter(|host| !host.is_empty()) + .map(str::to_string) +} + +#[derive(Debug, Clone)] +struct ProxyPodResourceNames { + supervisor_deployment: String, + service: String, + proxy_ca_secret: String, + agent_egress_network_policy: String, + supervisor_ingress_network_policy: String, +} + +fn proxy_pod_resource_names(sandbox_name: &str) -> ProxyPodResourceNames { + ProxyPodResourceNames { + supervisor_deployment: dns_label_name("os-sup", sandbox_name), + service: dns_label_name("os-svc", sandbox_name), + proxy_ca_secret: dns_label_name("os-ca", sandbox_name), + agent_egress_network_policy: dns_label_name("os-eg", sandbox_name), + supervisor_ingress_network_policy: dns_label_name("os-ing", sandbox_name), + } +} + +fn dns_label_name(prefix: &str, name: &str) -> String { + let mut hash = 0xcbf2_9ce4_8422_2325_u64; + for byte in name.as_bytes() { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + let suffix_hash = hash & 0xffff_ffff; + let suffix = format!("{suffix_hash:08x}"); + let mut sanitized = name + .chars() + .map(|c| { + let c = c.to_ascii_lowercase(); + if c.is_ascii_alphanumeric() || c == '-' { + c + } else { + '-' + } + }) + .collect::(); + sanitized = sanitized + .trim_matches('-') + .split('-') + .filter(|part| !part.is_empty()) + .collect::>() + .join("-"); + if sanitized.is_empty() { + sanitized = "sandbox".to_string(); + } + let max_base_len = 63usize.saturating_sub(prefix.len() + suffix.len() + 2); + if sanitized.len() > max_base_len { + sanitized.truncate(max_base_len); + sanitized = sanitized.trim_matches('-').to_string(); + } + format!("{prefix}-{sanitized}-{suffix}") +} + +fn proxy_pod_service_dns(service_name: &str, namespace: &str) -> String { + format!("{service_name}.{namespace}.svc.cluster.local") +} + +fn proxy_pod_process_gateway_endpoint(service_dns: &str, grpc_endpoint: &str) -> String { + if grpc_endpoint.is_empty() { + String::new() + } else if grpc_endpoint.starts_with("https://") { + format!("https://{service_dns}:{PROXY_POD_GATEWAY_FORWARD_PORT}") + } else { + format!("http://{service_dns}:{PROXY_POD_GATEWAY_FORWARD_PORT}") + } +} + +fn proxy_pod_proxy_url(service_dns: &str) -> String { + format!("http://{service_dns}:{PROXY_POD_PROXY_PORT}") +} + +fn apply_host_gateway_aliases( + spec: &mut serde_json::Map, + host_gateway_ip: &str, +) { + if host_gateway_ip.is_empty() { + return; + } + spec.insert( + "hostAliases".to_string(), + serde_json::json!([{ + "ip": host_gateway_ip, + "hostnames": ["host.docker.internal", "host.openshell.internal"] + }]), + ); +} + fn copy_log_level_env( env: &mut Vec, template_environment: &std::collections::HashMap, @@ -3034,86 +3349,369 @@ fn apply_supervisor_sidecar_topology( )); } -/// Apply workspace persistence transforms to an already-built pod template. -/// -/// This injects: -/// 1. A volume mount on the agent container at `/sandbox`. -/// 2. An init container (same image) that seeds the PVC with the image's -/// original `/sandbox` contents on first use. -/// -/// The PVC volume itself is **not** added here — the Sandbox CRD controller -/// automatically creates a volume for each entry in `volumeClaimTemplates` -/// (following the `StatefulSet` convention). Adding one here would create a -/// duplicate volume name and fail pod validation. -/// -/// The init container mounts the PVC at a temporary path so it can still see -/// the image's `/sandbox` directory. It checks for a sentinel file and skips -/// the copy if the PVC was already initialised. -#[allow(clippy::similar_names)] -fn apply_workspace_persistence( - pod_template: &mut serde_json::Value, +fn proxy_pod_ca_source_volume_mount() -> serde_json::Value { + serde_json::json!({ + "name": "openshell-proxy-pod-ca-source", + "mountPath": PROXY_POD_CA_SECRET_MOUNT_PATH, + "readOnly": true + }) +} + +fn proxy_pod_ca_tls_volume_mount() -> serde_json::Value { + serde_json::json!({ + "name": "openshell-proxy-pod-tls", + "mountPath": SIDECAR_TLS_MOUNT_PATH, + }) +} + +fn proxy_pod_ca_init_container( image: &str, image_pull_policy: &str, sandbox_gid: u32, +) -> serde_json::Value { + let copy_cmd = format!( + "set -eu; \ + mkdir -p {SIDECAR_TLS_MOUNT_PATH}; \ + cp {PROXY_POD_CA_SECRET_MOUNT_PATH}/{PROXY_POD_CA_CERT_FILE} {SIDECAR_TLS_MOUNT_PATH}/{PROXY_POD_CA_CERT_FILE}; \ + bundle={SIDECAR_TLS_MOUNT_PATH}/ca-bundle.pem; \ + found=0; \ + for path in /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/ca-bundle.pem /etc/ssl/cert.pem; do \ + if [ -f \"$path\" ]; then cat \"$path\" > \"$bundle\"; found=1; break; fi; \ + done; \ + if [ \"$found\" = 0 ]; then : > \"$bundle\"; fi; \ + printf '\\n' >> \"$bundle\"; \ + cat {PROXY_POD_CA_SECRET_MOUNT_PATH}/{PROXY_POD_CA_CERT_FILE} >> \"$bundle\"" + ); + let mut init_spec = serde_json::json!({ + "name": "openshell-proxy-ca-install", + "image": image, + "command": ["sh", "-c", copy_cmd], + "securityContext": { + "runAsUser": 0, + "runAsGroup": sandbox_gid, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + } + }, + "volumeMounts": [ + proxy_pod_ca_source_volume_mount(), + proxy_pod_ca_tls_volume_mount(), + ] + }); + if !image_pull_policy.is_empty() { + init_spec["imagePullPolicy"] = serde_json::json!(image_pull_policy); + } + init_spec +} + +fn apply_proxy_pod_affinity( + spec: &mut serde_json::Map, + sandbox_id: &str, +) { + if sandbox_id.is_empty() { + return; + } + + let affinity = spec + .entry("affinity".to_string()) + .or_insert_with(|| serde_json::json!({})); + if !affinity.is_object() { + *affinity = serde_json::json!({}); + } + let affinity = affinity + .as_object_mut() + .expect("affinity was converted to object"); + let pod_affinity = affinity + .entry("podAffinity".to_string()) + .or_insert_with(|| serde_json::json!({})); + if !pod_affinity.is_object() { + *pod_affinity = serde_json::json!({}); + } + let pod_affinity = pod_affinity + .as_object_mut() + .expect("podAffinity was converted to object"); + let required = pod_affinity + .entry("requiredDuringSchedulingIgnoredDuringExecution".to_string()) + .or_insert_with(|| serde_json::json!([])); + if !required.is_array() { + *required = serde_json::json!([]); + } + if let Some(required) = required.as_array_mut() { + required.push(serde_json::json!({ + "labelSelector": { + "matchLabels": proxy_pod_match_labels(sandbox_id, SANDBOX_ROLE_SUPERVISOR) + }, + "topologyKey": "kubernetes.io/hostname" + })); + } +} + +fn apply_supervisor_proxy_pod_topology( + pod_template: &mut serde_json::Value, + params: &SandboxPodParams<'_>, ) { let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { return; }; - // fsGroup is a pod-level field — it instructs kubelet to chown mounted - // volumes to this GID. It is invalid at the container securityContext level. - let pod_sc = spec + let pod_security_context = spec .entry("securityContext") .or_insert_with(|| serde_json::json!({})); - if let Some(pod_sc_obj) = pod_sc.as_object_mut() { - pod_sc_obj.insert("fsGroup".to_string(), serde_json::json!(sandbox_gid)); + if let Some(sc) = pod_security_context.as_object_mut() { + sc.insert("fsGroup".to_string(), serde_json::json!(params.sandbox_gid)); } - // 1. Add workspace volume mount to the agent container - let containers = spec.get_mut("containers").and_then(|v| v.as_array_mut()); - if let Some(containers) = containers { - let mut target_index = None; - for (i, c) in containers.iter().enumerate() { - if c.get("name").and_then(|v| v.as_str()) == Some("agent") { - target_index = Some(i); - break; - } - } - let index = target_index.unwrap_or(0); + apply_supervisor_binary_source( + spec, + params.supervisor_image, + params.supervisor_image_pull_policy, + params.supervisor_sideload_method, + ); - if let Some(container) = containers.get_mut(index).and_then(|v| v.as_object_mut()) { - let volume_mounts = container - .entry("volumeMounts") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volume_mounts) = volume_mounts { - volume_mounts.push(serde_json::json!({ - "name": WORKSPACE_VOLUME_NAME, - "mountPath": WORKSPACE_MOUNT_PATH - })); + apply_proxy_pod_affinity(spec, params.sandbox_id); + + let names = proxy_pod_resource_names(params.sandbox_name); + let service_dns = proxy_pod_service_dns(&names.service, params.namespace); + + let volumes = spec + .entry("volumes") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volumes) = volumes { + volumes.push(serde_json::json!({ + "name": "openshell-proxy-pod-ca-source", + "secret": { + "secretName": names.proxy_ca_secret, + "defaultMode": 0o444, + "items": [{ + "key": PROXY_POD_CA_CERT_FILE, + "path": PROXY_POD_CA_CERT_FILE, + }] } - } + })); + volumes.push(serde_json::json!({ + "name": "openshell-proxy-pod-tls", + "emptyDir": {} + })); } - // 3. Add the init container that seeds the PVC from the image + let image = spec + .get("containers") + .and_then(|v| v.as_array()) + .and_then(|containers| containers.first()) + .and_then(|container| container.get("image")) + .and_then(|value| value.as_str()) + .unwrap_or(params.default_image) + .to_string(); let init_containers = spec .entry("initContainers") .or_insert_with(|| serde_json::json!([])) .as_array_mut(); if let Some(init_containers) = init_containers { - // The init container mounts the PVC at a temp path so it can still - // read the image's original /sandbox contents. It copies them into - // the PVC only when the sentinel file is absent. - // - // Prefer a tar stream over `cp -a`: some sandbox images contain - // self-referential symlinks under `/sandbox/.uv`, and GNU cp can - // fail while seeding the PVC even though preserving the symlink as-is - // is valid. `tar` copies the tree without dereferencing those links. - // Archive only the contents, not the `/sandbox` directory entry - // itself, so extraction never tries to chmod the PVC mount root. - // Extract without restoring owner, mode, or timestamps so the - // non-root init container can seed kubelet-owned PVCs. - // + init_containers.push(proxy_pod_ca_init_container( + &image, + params.image_pull_policy, + params.sandbox_gid, + )); + } + + let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { + return; + }; + let target_index = containers + .iter() + .position(|c| c.get("name").and_then(|v| v.as_str()) == Some("agent")) + .unwrap_or(0); + if let Some(container) = containers + .get_mut(target_index) + .and_then(|v| v.as_object_mut()) + { + container.insert( + "command".to_string(), + serde_json::json!([ + format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), + "--mode=process" + ]), + ); + + let security_context = container + .entry("securityContext") + .or_insert_with(|| serde_json::json!({})); + if let Some(sc) = security_context.as_object_mut() { + sc.insert( + "runAsUser".to_string(), + serde_json::json!(params.sandbox_uid), + ); + sc.insert( + "runAsGroup".to_string(), + serde_json::json!(params.sandbox_gid), + ); + sc.insert("runAsNonRoot".to_string(), serde_json::json!(true)); + sc.insert( + "allowPrivilegeEscalation".to_string(), + serde_json::json!(false), + ); + sc.insert( + "capabilities".to_string(), + serde_json::json!({ + "drop": ["ALL"] + }), + ); + } + + let volume_mounts = container + .entry("volumeMounts") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volume_mounts) = volume_mounts { + volume_mounts.push(supervisor_volume_mount()); + volume_mounts.push(proxy_pod_ca_tls_volume_mount()); + } + + let env = container + .entry("env") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(env) = env { + let process_endpoint = + proxy_pod_process_gateway_endpoint(&service_dns, params.grpc_endpoint); + upsert_env( + env, + openshell_core::sandbox_env::ENDPOINT, + &process_endpoint, + ); + if let Some(server_name) = gateway_tls_server_name(params.grpc_endpoint) { + upsert_env( + env, + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + &server_name, + ); + } + upsert_env( + env, + openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, + "proxy-pod", + ); + upsert_env( + env, + openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, + PROXY_POD_NETWORK_ENFORCEMENT_MODE, + ); + upsert_env( + env, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + PROXY_POD_SSH_SOCKET_FILE, + ); + upsert_env( + env, + openshell_core::sandbox_env::PROXY_URL, + &proxy_pod_proxy_url(&service_dns), + ); + upsert_env( + env, + openshell_core::sandbox_env::SUPERVISOR_READY_ADDR, + &format!("{service_dns}:{PROXY_POD_PROXY_PORT}"), + ); + upsert_env( + env, + openshell_core::sandbox_env::PROXY_TLS_DIR, + SIDECAR_TLS_MOUNT_PATH, + ); + upsert_env( + env, + openshell_core::sandbox_env::SANDBOX_UID, + ¶ms.sandbox_uid.to_string(), + ); + upsert_env( + env, + openshell_core::sandbox_env::SANDBOX_GID, + ¶ms.sandbox_gid.to_string(), + ); + } + } +} + +/// Apply workspace persistence transforms to an already-built pod template. +/// +/// This injects: +/// 1. A volume mount on the agent container at `/sandbox`. +/// 2. An init container (same image) that seeds the PVC with the image's +/// original `/sandbox` contents on first use. +/// +/// The PVC volume itself is **not** added here — the Sandbox CRD controller +/// automatically creates a volume for each entry in `volumeClaimTemplates` +/// (following the `StatefulSet` convention). Adding one here would create a +/// duplicate volume name and fail pod validation. +/// +/// The init container mounts the PVC at a temporary path so it can still see +/// the image's `/sandbox` directory. It checks for a sentinel file and skips +/// the copy if the PVC was already initialised. +#[allow(clippy::similar_names)] +fn apply_workspace_persistence( + pod_template: &mut serde_json::Value, + image: &str, + image_pull_policy: &str, + sandbox_gid: u32, +) { + let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { + return; + }; + + // fsGroup is a pod-level field — it instructs kubelet to chown mounted + // volumes to this GID. It is invalid at the container securityContext level. + let pod_sc = spec + .entry("securityContext") + .or_insert_with(|| serde_json::json!({})); + if let Some(pod_sc_obj) = pod_sc.as_object_mut() { + pod_sc_obj.insert("fsGroup".to_string(), serde_json::json!(sandbox_gid)); + } + + // 1. Add workspace volume mount to the agent container + let containers = spec.get_mut("containers").and_then(|v| v.as_array_mut()); + if let Some(containers) = containers { + let mut target_index = None; + for (i, c) in containers.iter().enumerate() { + if c.get("name").and_then(|v| v.as_str()) == Some("agent") { + target_index = Some(i); + break; + } + } + let index = target_index.unwrap_or(0); + + if let Some(container) = containers.get_mut(index).and_then(|v| v.as_object_mut()) { + let volume_mounts = container + .entry("volumeMounts") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volume_mounts) = volume_mounts { + volume_mounts.push(serde_json::json!({ + "name": WORKSPACE_VOLUME_NAME, + "mountPath": WORKSPACE_MOUNT_PATH + })); + } + } + } + + // 3. Add the init container that seeds the PVC from the image + let init_containers = spec + .entry("initContainers") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(init_containers) = init_containers { + // The init container mounts the PVC at a temp path so it can still + // read the image's original /sandbox contents. It copies them into + // the PVC only when the sentinel file is absent. + // + // Prefer a tar stream over `cp -a`: some sandbox images contain + // self-referential symlinks under `/sandbox/.uv`, and GNU cp can + // fail while seeding the PVC even though preserving the symlink as-is + // is valid. `tar` copies the tree without dereferencing those links. + // Archive only the contents, not the `/sandbox` directory entry + // itself, so extraction never tries to chmod the PVC mount root. + // Extract without restoring owner, mode, or timestamps so the + // non-root init container can seed kubelet-owned PVCs. + // // The inner `[ -d ... ]` guard handles custom images that don't have // a /sandbox directory — the copy is skipped but the sentinel is // still written so subsequent starts are instant. @@ -3205,6 +3803,7 @@ struct SandboxPodParams<'a> { proxy_auth_secret_key: Option<&'a str>, proxy_auth_allow_insecure: bool, proxy_connect_by_hostname: bool, + namespace: &'a str, service_account_name: &'a str, sandbox_id: &'a str, sandbox_name: &'a str, @@ -3246,6 +3845,7 @@ impl Default for SandboxPodParams<'_> { proxy_auth_secret_key: None, proxy_auth_allow_insecure: false, proxy_connect_by_hostname: false, + namespace: "default", service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, sandbox_id: "", sandbox_name: "", @@ -3267,12 +3867,15 @@ impl Default for SandboxPodParams<'_> { } } -fn validate_sidecar_proxy_identity( - params: &SandboxPodParams<'_>, -) -> Result<(), KubernetesDriverError> { - if params.topology == SupervisorTopology::Sidecar && params.proxy_uid == params.sandbox_uid { +fn validate_proxy_identity(params: &SandboxPodParams<'_>) -> Result<(), KubernetesDriverError> { + if matches!( + params.topology, + SupervisorTopology::Sidecar | SupervisorTopology::ProxyPod + ) && params.proxy_uid == params.sandbox_uid + { + let topology = params.topology.to_string(); return Err(KubernetesDriverError::Precondition(format!( - "proxy_uid ({}) must not match sandbox_uid ({}) in sidecar topology", + "proxy_uid ({}) must not match sandbox_uid ({}) in {topology} topology", params.proxy_uid, params.sandbox_uid ))); } @@ -3442,7 +4045,8 @@ fn sandbox_template_to_k8s_with_validated_config( .iter() .map(|(key, value)| (key.clone(), serde_json::Value::String(value.clone()))) .collect::>(); - if params.provider_spiffe_enabled { + let proxy_pod_topology = params.topology == SupervisorTopology::ProxyPod; + if params.provider_spiffe_enabled || proxy_pod_topology { pod_labels.insert( LABEL_MANAGED_BY.to_string(), serde_json::Value::String(LABEL_MANAGED_BY_VALUE.to_string()), @@ -3454,6 +4058,12 @@ fn sandbox_template_to_k8s_with_validated_config( ); } } + if proxy_pod_topology { + pod_labels.insert( + LABEL_SANDBOX_ROLE.to_string(), + serde_json::Value::String(SANDBOX_ROLE_AGENT.to_string()), + ); + } if !pod_labels.is_empty() { metadata.insert("labels".to_string(), serde_json::Value::Object(pod_labels)); } @@ -3650,7 +4260,7 @@ fn sandbox_template_to_k8s_with_validated_config( if !params.client_tls_secret_name.is_empty() { let client_tls_default_mode = match params.topology { SupervisorTopology::Combined => 0o400, - SupervisorTopology::Sidecar => 0o440, + SupervisorTopology::Sidecar | SupervisorTopology::ProxyPod => 0o440, }; volumes.push(serde_json::json!({ "name": CLIENT_TLS_VOLUME_NAME, @@ -3671,7 +4281,9 @@ fn sandbox_template_to_k8s_with_validated_config( // network supervision. Sidecar mode uses the pod fsGroup already // required for its non-root network supervisor. let default_mode = match params.topology { - SupervisorTopology::Combined => 0o400, + // `combined` and `proxy-pod` are rejected by + // `validate_upstream_proxy_config`; use the most restrictive mode. + SupervisorTopology::Combined | SupervisorTopology::ProxyPod => 0o400, SupervisorTopology::Sidecar => 0o440, }; volumes.push(serde_json::json!({ @@ -3702,7 +4314,7 @@ fn sandbox_template_to_k8s_with_validated_config( // supervisor containers run with the sandbox GID and need group-read access. let sa_token_default_mode = match params.topology { SupervisorTopology::Combined => 0o400, - SupervisorTopology::Sidecar => 0o440, + SupervisorTopology::Sidecar | SupervisorTopology::ProxyPod => 0o440, }; volumes.push(serde_json::json!({ "name": SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, @@ -3726,15 +4338,7 @@ fn sandbox_template_to_k8s_with_validated_config( spec.insert("volumes".to_string(), serde_json::Value::Array(volumes)); // Add hostAliases so sandbox pods can reach the Docker host. - if !params.host_gateway_ip.is_empty() { - spec.insert( - "hostAliases".to_string(), - serde_json::json!([{ - "ip": params.host_gateway_ip, - "hostnames": ["host.docker.internal", "host.openshell.internal"] - }]), - ); - } + apply_host_gateway_aliases(&mut spec, params.host_gateway_ip); let mut template_value = serde_json::Map::new(); if !metadata.is_empty() { @@ -3756,6 +4360,9 @@ fn sandbox_template_to_k8s_with_validated_config( params, ); } + SupervisorTopology::ProxyPod => { + apply_supervisor_proxy_pod_topology(&mut result, params); + } } // Inject workspace persistence (init container + PVC volume mount) so @@ -3784,79 +4391,582 @@ fn apply_pod_driver_config( merge_string_map(node_selector, &config.node_selector); } - if !config.priority_class_name.is_empty() { - spec.entry("priorityClassName".to_string()) - .or_insert_with(|| serde_json::json!(config.priority_class_name)); + if !config.priority_class_name.is_empty() { + spec.entry("priorityClassName".to_string()) + .or_insert_with(|| serde_json::json!(config.priority_class_name)); + } + + if !config.tolerations.is_empty() { + let tolerations = spec + .entry("tolerations".to_string()) + .or_insert_with(|| serde_json::json!([])); + if let Some(existing) = tolerations.as_array_mut() { + existing.extend(config.tolerations.iter().cloned()); + } else { + *tolerations = serde_json::Value::Array(config.tolerations.clone()); + } + } +} + +fn apply_agent_driver_resources( + container: &mut serde_json::Map, + resources: &KubernetesContainerResourceConfig, +) { + if resources.requests.is_empty() && resources.limits.is_empty() { + return; + } + + let target = container + .entry("resources".to_string()) + .or_insert_with(|| serde_json::json!({})); + apply_resource_quantity_map(target, "requests", &resources.requests); + apply_resource_quantity_map(target, "limits", &resources.limits); +} + +fn merge_string_map(target: &mut serde_json::Value, values: &BTreeMap) { + if !target.is_object() { + *target = serde_json::json!({}); + } + let target = target + .as_object_mut() + .expect("target was converted to object"); + for (key, value) in values { + target + .entry(key.clone()) + .or_insert_with(|| serde_json::json!(value)); + } +} + +fn apply_resource_quantity_map( + target: &mut serde_json::Value, + section: &str, + values: &BTreeMap, +) { + if values.is_empty() { + return; + } + if !target.is_object() { + *target = serde_json::json!({}); + } + let target = target + .as_object_mut() + .expect("target was converted to object"); + let section_value = target + .entry(section.to_string()) + .or_insert_with(|| serde_json::json!({})); + merge_string_map(section_value, values); +} + +fn image_pull_secret_refs(secrets: &[String]) -> Vec { + secrets + .iter() + .map(|secret| secret.trim()) + .filter(|secret| !secret.is_empty()) + .map(|secret| serde_json::json!({ "name": secret })) + .collect() +} + +fn k8s_object(value: serde_json::Value) -> T +where + T: DeserializeOwned, +{ + serde_json::from_value(value).expect("driver rendered an invalid Kubernetes object") +} + +fn generate_proxy_pod_ca() -> Result<(String, String), KubernetesDriverError> { + let ca_key = KeyPair::generate().map_err(|err| { + KubernetesDriverError::Message(format!("failed to generate CA key: {err}")) + })?; + + let mut params = CertificateParams::default(); + params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + params + .distinguished_name + .push(DnType::CommonName, "OpenShell Proxy Pod Sandbox CA"); + params + .distinguished_name + .push(DnType::OrganizationName, "OpenShell"); + params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; + + let ca_cert = params.self_signed(&ca_key).map_err(|err| { + KubernetesDriverError::Message(format!("failed to generate CA certificate: {err}")) + })?; + Ok((ca_cert.pem(), ca_key.serialize_pem())) +} + +fn proxy_pod_owner_reference( + sandbox_cr: &DynamicObject, + api_version: &str, + controller: bool, +) -> Result { + let name = + sandbox_cr.metadata.name.as_deref().ok_or_else(|| { + KubernetesDriverError::Message("created Sandbox is missing name".into()) + })?; + let uid = + sandbox_cr.metadata.uid.as_deref().ok_or_else(|| { + KubernetesDriverError::Message("created Sandbox is missing uid".into()) + })?; + Ok(serde_json::json!({ + "apiVersion": sandbox_cr + .types + .as_ref() + .map_or(api_version, |types| types.api_version.as_str()), + "kind": SANDBOX_KIND, + "name": name, + "uid": uid, + "controller": controller, + "blockOwnerDeletion": false, + })) +} + +fn proxy_pod_labels(sandbox_id: &str, role: &str) -> serde_json::Value { + let mut labels = serde_json::Map::new(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + serde_json::json!(LABEL_MANAGED_BY_VALUE), + ); + labels.insert(LABEL_SANDBOX_ID.to_string(), serde_json::json!(sandbox_id)); + labels.insert(LABEL_SANDBOX_ROLE.to_string(), serde_json::json!(role)); + serde_json::Value::Object(labels) +} + +fn proxy_pod_match_labels(sandbox_id: &str, role: &str) -> serde_json::Value { + let mut labels = serde_json::Map::new(); + labels.insert(LABEL_SANDBOX_ID.to_string(), serde_json::json!(sandbox_id)); + labels.insert(LABEL_SANDBOX_ROLE.to_string(), serde_json::json!(role)); + serde_json::Value::Object(labels) +} + +fn proxy_pod_object_meta( + name: &str, + namespace: &str, + sandbox_id: &str, + role: &str, + owner_ref: serde_json::Value, +) -> serde_json::Value { + serde_json::json!({ + "name": name, + "namespace": namespace, + "labels": proxy_pod_labels(sandbox_id, role), + "annotations": { + "openshell.io/sandbox-id": sandbox_id + }, + "ownerReferences": [owner_ref] + }) +} + +fn proxy_pod_supervisor_env( + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + params: &SandboxPodParams<'_>, +) -> Vec { + let mut env = Vec::new(); + apply_required_env( + &mut env, + params.sandbox_id, + params.sandbox_name, + params.grpc_endpoint, + "", + false, + provider_spiffe_socket_path(params), + ); + if !params.client_tls_secret_name.is_empty() { + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_CA, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/ca.crt"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_CERT, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.crt"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_KEY, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.key"), + ); + } + copy_log_level_env(&mut env, template_environment, spec_environment); + upsert_env( + &mut env, + openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, + "proxy-pod", + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, + PROXY_POD_NETWORK_ENFORCEMENT_MODE, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, + "relaxed", + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR, + PROXY_POD_GATEWAY_FORWARD_ADDR, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::PROXY_BIND_ADDR, + &format!("0.0.0.0:{PROXY_POD_PROXY_PORT}"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::PROXY_TLS_DIR, + SIDECAR_TLS_MOUNT_PATH, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::PROXY_CA_CERT_PATH, + &format!("{PROXY_POD_CA_SECRET_MOUNT_PATH}/{PROXY_POD_CA_CERT_FILE}"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::PROXY_CA_KEY_PATH, + &format!("{PROXY_POD_CA_SECRET_MOUNT_PATH}/{PROXY_POD_CA_KEY_FILE}"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SANDBOX_UID, + ¶ms.sandbox_uid.to_string(), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SANDBOX_GID, + ¶ms.sandbox_gid.to_string(), + ); + env +} + +fn proxy_pod_ca_secret( + names: &ProxyPodResourceNames, + params: &SandboxPodParams<'_>, + owner_ref: serde_json::Value, + cert_pem: &str, + key_pem: &str, +) -> Secret { + let mut string_data = serde_json::Map::new(); + string_data.insert( + PROXY_POD_CA_CERT_FILE.to_string(), + serde_json::json!(cert_pem), + ); + string_data.insert( + PROXY_POD_CA_KEY_FILE.to_string(), + serde_json::json!(key_pem), + ); + k8s_object(serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": names.proxy_ca_secret, + "namespace": params.namespace, + "labels": proxy_pod_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR), + "ownerReferences": [owner_ref], + }, + "type": "Opaque", + "stringData": serde_json::Value::Object(string_data) + })) +} + +fn proxy_pod_supervisor_service( + names: &ProxyPodResourceNames, + params: &SandboxPodParams<'_>, + owner_ref: serde_json::Value, +) -> Service { + k8s_object(serde_json::json!({ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": names.service, + "namespace": params.namespace, + "labels": proxy_pod_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR), + "ownerReferences": [owner_ref], + }, + "spec": { + "clusterIP": "None", + "publishNotReadyAddresses": true, + "selector": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR), + "ports": [ + { + "name": "http-proxy", + "port": PROXY_POD_PROXY_PORT, + "targetPort": PROXY_POD_PROXY_PORT, + "protocol": "TCP" + }, + { + "name": "gateway-forward", + "port": PROXY_POD_GATEWAY_FORWARD_PORT, + "targetPort": PROXY_POD_GATEWAY_FORWARD_PORT, + "protocol": "TCP" + } + ] + } + })) +} + +fn proxy_pod_supervisor_deployment( + names: &ProxyPodResourceNames, + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + params: &SandboxPodParams<'_>, + owner_ref: serde_json::Value, +) -> Deployment { + let mut container = serde_json::json!({ + "name": SUPERVISOR_NETWORK_SIDECAR_NAME, + "image": params.supervisor_image, + "command": [ + SUPERVISOR_IMAGE_BINARY_PATH, + "--mode=network", + ], + "env": proxy_pod_supervisor_env(template_environment, spec_environment, params), + "ports": [ + {"name": "http-proxy", "containerPort": PROXY_POD_PROXY_PORT, "protocol": "TCP"}, + {"name": "gateway-fwd", "containerPort": PROXY_POD_GATEWAY_FORWARD_PORT, "protocol": "TCP"} + ], + "readinessProbe": { + "tcpSocket": {"port": PROXY_POD_PROXY_PORT}, + "periodSeconds": 2, + "failureThreshold": 30 + }, + "securityContext": { + "runAsUser": params.proxy_uid, + "runAsGroup": params.sandbox_gid, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + } + }, + "volumeMounts": [ + { + "name": "openshell-sa-token", + "mountPath": "/var/run/secrets/openshell", + "readOnly": true + }, + { + "name": "openshell-proxy-pod-ca-source", + "mountPath": PROXY_POD_CA_SECRET_MOUNT_PATH, + "readOnly": true + }, + proxy_pod_ca_tls_volume_mount(), + ] + }); + if !params.supervisor_image_pull_policy.is_empty() { + container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + } + if !params.client_tls_secret_name.is_empty() { + container["volumeMounts"] + .as_array_mut() + .expect("volumeMounts is an array") + .push(serde_json::json!({ + "name": "openshell-client-tls", + "mountPath": SIDECAR_CLIENT_TLS_MOUNT_PATH, + "readOnly": true + })); + } + if params.provider_spiffe_enabled { + container["volumeMounts"] + .as_array_mut() + .expect("volumeMounts is an array") + .push(serde_json::json!({ + "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, + "mountPath": spiffe_socket_mount_path(params.provider_spiffe_workload_api_socket_path), + "readOnly": true, + })); + } + if let Some(profile) = params.app_armor_profile { + container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); + } + + let mut spec = serde_json::json!({ + "serviceAccountName": params.service_account_name, + "automountServiceAccountToken": false, + "securityContext": { + "fsGroup": params.sandbox_gid + }, + "containers": [container], + "volumes": [ + { + "name": "openshell-sa-token", + "projected": { + "sources": [{ + "serviceAccountToken": { + "audience": "openshell-gateway", + "expirationSeconds": params.sa_token_ttl_secs, + "path": "token" + } + }], + "defaultMode": 0o440 + } + }, + { + "name": "openshell-proxy-pod-ca-source", + "secret": { + "secretName": names.proxy_ca_secret, + "defaultMode": 0o440 + } + }, + { + "name": "openshell-proxy-pod-tls", + "emptyDir": {} + } + ] + }); + if !params.default_runtime_class_name.is_empty() { + spec["runtimeClassName"] = serde_json::json!(params.default_runtime_class_name); } - - if !config.tolerations.is_empty() { - let tolerations = spec - .entry("tolerations".to_string()) - .or_insert_with(|| serde_json::json!([])); - if let Some(existing) = tolerations.as_array_mut() { - existing.extend(config.tolerations.iter().cloned()); - } else { - *tolerations = serde_json::Value::Array(config.tolerations.clone()); - } + if let Some(spec_obj) = spec.as_object_mut() { + apply_host_gateway_aliases(spec_obj, params.host_gateway_ip); } -} - -fn apply_agent_driver_resources( - container: &mut serde_json::Map, - resources: &KubernetesContainerResourceConfig, -) { - if resources.requests.is_empty() && resources.limits.is_empty() { - return; + let image_pull_secrets = image_pull_secret_refs(params.image_pull_secrets); + if !image_pull_secrets.is_empty() { + spec["imagePullSecrets"] = serde_json::Value::Array(image_pull_secrets); } - - let target = container - .entry("resources".to_string()) - .or_insert_with(|| serde_json::json!({})); - apply_resource_quantity_map(target, "requests", &resources.requests); - apply_resource_quantity_map(target, "limits", &resources.limits); -} - -fn merge_string_map(target: &mut serde_json::Value, values: &BTreeMap) { - if !target.is_object() { - *target = serde_json::json!({}); + if !params.client_tls_secret_name.is_empty() { + spec["volumes"] + .as_array_mut() + .expect("volumes is an array") + .push(serde_json::json!({ + "name": "openshell-client-tls", + "secret": { + "secretName": params.client_tls_secret_name, + "defaultMode": 0o440 + } + })); } - let target = target - .as_object_mut() - .expect("target was converted to object"); - for (key, value) in values { - target - .entry(key.clone()) - .or_insert_with(|| serde_json::json!(value)); + if params.provider_spiffe_enabled { + spec["volumes"] + .as_array_mut() + .expect("volumes is an array") + .push(serde_json::json!({ + "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, + "csi": { + "driver": "csi.spiffe.io", + "readOnly": true + } + })); } + + k8s_object(serde_json::json!({ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": proxy_pod_object_meta( + &names.supervisor_deployment, + params.namespace, + params.sandbox_id, + SANDBOX_ROLE_SUPERVISOR, + owner_ref + ), + "spec": { + "replicas": 1, + "selector": { + "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR) + }, + "template": { + "metadata": { + "labels": proxy_pod_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR), + "annotations": { + "openshell.io/sandbox-id": params.sandbox_id + } + }, + "spec": spec + } + } + })) } -fn apply_resource_quantity_map( - target: &mut serde_json::Value, - section: &str, - values: &BTreeMap, -) { - if values.is_empty() { - return; - } - if !target.is_object() { - *target = serde_json::json!({}); - } - let target = target - .as_object_mut() - .expect("target was converted to object"); - let section_value = target - .entry(section.to_string()) - .or_insert_with(|| serde_json::json!({})); - merge_string_map(section_value, values); +fn proxy_pod_agent_egress_network_policy( + names: &ProxyPodResourceNames, + params: &SandboxPodParams<'_>, + owner_ref: serde_json::Value, +) -> NetworkPolicy { + k8s_object(serde_json::json!({ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": names.agent_egress_network_policy, + "namespace": params.namespace, + "labels": proxy_pod_labels(params.sandbox_id, SANDBOX_ROLE_AGENT), + "ownerReferences": [owner_ref], + }, + "spec": { + "podSelector": { + "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_AGENT) + }, + "policyTypes": ["Egress"], + "egress": [ + { + "to": [{ + "podSelector": { + "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR) + } + }], + "ports": [ + {"protocol": "TCP", "port": PROXY_POD_PROXY_PORT}, + {"protocol": "TCP", "port": PROXY_POD_GATEWAY_FORWARD_PORT} + ] + }, + { + "to": [{ + "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}, + "podSelector": {"matchLabels": {"k8s-app": "kube-dns"}} + }], + "ports": [ + {"protocol": "UDP", "port": 53}, + {"protocol": "TCP", "port": 53} + ] + }, + { + "to": [{ + "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}, + "podSelector": {"matchLabels": {"k8s-app": "coredns"}} + }], + "ports": [ + {"protocol": "UDP", "port": 53}, + {"protocol": "TCP", "port": 53} + ] + } + ] + } + })) } -fn image_pull_secret_refs(secrets: &[String]) -> Vec { - secrets - .iter() - .map(|secret| secret.trim()) - .filter(|secret| !secret.is_empty()) - .map(|secret| serde_json::json!({ "name": secret })) - .collect() +fn proxy_pod_supervisor_ingress_network_policy( + names: &ProxyPodResourceNames, + params: &SandboxPodParams<'_>, + owner_ref: serde_json::Value, +) -> NetworkPolicy { + k8s_object(serde_json::json!({ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": names.supervisor_ingress_network_policy, + "namespace": params.namespace, + "labels": proxy_pod_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR), + "ownerReferences": [owner_ref], + }, + "spec": { + "podSelector": { + "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR) + }, + "policyTypes": ["Ingress"], + "ingress": [{ + "from": [{ + "podSelector": { + "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_AGENT) + } + }], + "ports": [ + {"protocol": "TCP", "port": PROXY_POD_PROXY_PORT}, + {"protocol": "TCP", "port": PROXY_POD_GATEWAY_FORWARD_PORT} + ] + }] + } + })) } fn app_armor_profile_to_k8s(profile: &AppArmorProfile) -> serde_json::Value { @@ -5708,6 +6818,7 @@ mod tests { grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", client_tls_secret_name: "openshell-client-tls", proxy_uid: 2200, + namespace: "default", sandbox_uid: 1500, sandbox_gid: 1500, ..SandboxPodParams::default() @@ -6092,15 +7203,227 @@ mod tests { let params = SandboxPodParams { topology: SupervisorTopology::Sidecar, proxy_uid: 1500, + namespace: "default", sandbox_uid: 1500, ..SandboxPodParams::default() }; - let err = validate_sidecar_proxy_identity(¶ms).unwrap_err(); + let err = validate_proxy_identity(¶ms).unwrap_err(); assert!(matches!(err, KubernetesDriverError::Precondition(_))); assert!(err.to_string().contains("proxy_uid")); } + #[test] + fn proxy_pod_topology_renders_process_agent_with_proxy_service() { + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, + supervisor_image: "supervisor-image:latest", + namespace: "agents", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1500, + host_gateway_ip: "172.17.0.1", + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + + let names = proxy_pod_resource_names("example-sandbox"); + let service_dns = proxy_pod_service_dns(&names.service, "agents"); + let agent = &pod_template["spec"]["containers"][0]; + + assert_eq!( + pod_template["metadata"]["labels"][LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_AGENT + ); + assert_eq!( + agent["command"], + serde_json::json!([ + format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), + "--mode=process" + ]) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::ENDPOINT), + Some(format!("https://{service_dns}:18080").as_str()) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME), + Some("openshell-gateway.openshell.svc") + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::PROXY_URL), + Some(format!("http://{service_dns}:3128").as_str()) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SUPERVISOR_READY_ADDR), + Some(format!("{service_dns}:3128").as_str()) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE), + Some(PROXY_POD_NETWORK_ENFORCEMENT_MODE) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SSH_SOCKET_PATH), + Some(PROXY_POD_SSH_SOCKET_FILE) + ); + + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + assert_eq!(containers.len(), 1); + let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); + assert!(volumes.iter().any(|volume| { + volume["name"] == "openshell-proxy-pod-ca-source" + && volume["secret"]["secretName"] == names.proxy_ca_secret + })); + assert!(volumes.iter().any(|volume| { + volume["name"] == "openshell-proxy-pod-tls" && volume["emptyDir"].is_object() + })); + + let affinity = &pod_template["spec"]["affinity"]["podAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] + [0]; + assert_eq!( + affinity["labelSelector"]["matchLabels"][LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_SUPERVISOR + ); + assert_eq!(affinity["topologyKey"], "kubernetes.io/hostname"); + } + + #[test] + fn proxy_pod_companion_resources_bind_one_agent_to_one_supervisor() { + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_image: "supervisor-image:latest", + namespace: "agents", + service_account_name: "openshell-sandbox", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + grpc_endpoint: "http://openshell-gateway.openshell.svc:8080", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1500, + host_gateway_ip: "172.17.0.1", + ..SandboxPodParams::default() + }; + let names = proxy_pod_resource_names(params.sandbox_name); + let owner_ref = serde_json::json!({ + "apiVersion": "agents.x-k8s.io/v1beta1", + "kind": "Sandbox", + "name": params.sandbox_name, + "uid": "sandbox-cr-uid", + "controller": true, + "blockOwnerDeletion": false + }); + + let supervisor = serde_json::to_value(proxy_pod_supervisor_deployment( + &names, + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + ¶ms, + owner_ref.clone(), + )) + .unwrap(); + assert_eq!( + supervisor["metadata"]["ownerReferences"][0]["controller"], + true + ); + assert_eq!( + supervisor["metadata"]["annotations"]["openshell.io/sandbox-id"], + "sandbox-123" + ); + assert_eq!( + supervisor["metadata"]["labels"][LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_SUPERVISOR + ); + assert_eq!(supervisor["kind"], "Deployment"); + assert_eq!(supervisor["spec"]["replicas"], 1); + assert_eq!( + supervisor["spec"]["selector"]["matchLabels"][LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_SUPERVISOR + ); + assert_eq!( + supervisor["spec"]["template"]["metadata"]["labels"][LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_SUPERVISOR + ); + assert_eq!( + supervisor["spec"]["template"]["spec"]["hostAliases"][0]["ip"], + params.host_gateway_ip + ); + let hostnames = supervisor["spec"]["template"]["spec"]["hostAliases"][0]["hostnames"] + .as_array() + .unwrap(); + assert!(hostnames.contains(&serde_json::json!("host.openshell.internal"))); + let container = &supervisor["spec"]["template"]["spec"]["containers"][0]; + assert_eq!( + rendered_env(container, openshell_core::sandbox_env::PROXY_BIND_ADDR), + Some("0.0.0.0:3128") + ); + assert_eq!( + rendered_env(container, openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR), + Some(PROXY_POD_GATEWAY_FORWARD_ADDR) + ); + + let agent_egress = serde_json::to_value(proxy_pod_agent_egress_network_policy( + &names, + ¶ms, + owner_ref.clone(), + )) + .unwrap(); + assert_eq!( + agent_egress["spec"]["policyTypes"], + serde_json::json!(["Egress"]) + ); + assert_eq!( + agent_egress["spec"]["podSelector"]["matchLabels"][LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_AGENT + ); + assert_eq!( + agent_egress["spec"]["egress"][0]["to"][0]["podSelector"]["matchLabels"] + [LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_SUPERVISOR + ); + + let supervisor_ingress = serde_json::to_value(proxy_pod_supervisor_ingress_network_policy( + &names, ¶ms, owner_ref, + )) + .unwrap(); + assert_eq!( + supervisor_ingress["spec"]["policyTypes"], + serde_json::json!(["Ingress"]) + ); + assert_eq!( + supervisor_ingress["spec"]["ingress"][0]["from"][0]["podSelector"]["matchLabels"] + [LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_AGENT + ); + } + + #[test] + fn proxy_pod_topology_rejects_proxy_uid_matching_sandbox_uid() { + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + proxy_uid: 1500, + namespace: "default", + sandbox_uid: 1500, + ..SandboxPodParams::default() + }; + + let err = validate_proxy_identity(¶ms).unwrap_err(); + assert!(matches!(err, KubernetesDriverError::Precondition(_))); + assert!(err.to_string().contains("proxy-pod")); + } + /// Regression test: TLS mount path must match env var paths. /// The volume is mounted at a specific path and the env vars must point to /// files within that same path, otherwise the sandbox will fail to start diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index d69f9749a1..f994a7663d 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -7,9 +7,9 @@ pub mod grpc; pub use config::{ AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, - ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, - managed_namespace_prefix, + DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesProxyPodConfig, + KubernetesSidecarConfig, ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, + WorkspaceMode, managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 3a805c8685..cc7990558d 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -14,8 +14,8 @@ use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServ use openshell_driver_kubernetes::{ AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, - KubernetesSidecarConfig, ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, - WorkspaceMode, + KubernetesProxyPodConfig, KubernetesSidecarConfig, ManagedSshIngressConfig, + SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, }; #[derive(Parser, Debug)] @@ -161,6 +161,14 @@ struct Args { #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME", action = ArgAction::SetTrue)] proxy_connect_by_hostname: bool, + /// UID for the proxy container in `proxy-pod` topology. + #[arg( + long = "proxy-pod-proxy-uid", + env = "OPENSHELL_K8S_PROXY_POD_PROXY_UID", + default_value_t = DEFAULT_PROXY_UID + )] + proxy_pod_proxy_uid: u32, + #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, @@ -257,6 +265,9 @@ async fn main() -> Result<()> { process_binary_aware_network_policy: args .sidecar_process_binary_aware_network_policy, }, + proxy_pod: KubernetesProxyPodConfig { + proxy_uid: args.proxy_pod_proxy_uid, + }, https_proxy: args.https_proxy, no_proxy: args.no_proxy, proxy_auth_secret_name: args.proxy_auth_secret_name, diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index d96f141cc8..9dd10a736b 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -65,11 +65,14 @@ use openshell_supervisor_network::opa::OpaEngine; use openshell_supervisor_process::process::ProcessEnforcementMode; pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; use openshell_supervisor_process::skills; +use tokio::io::copy_bidirectional; +use tokio::net::{TcpListener, TcpStream}; use tokio::sync::mpsc::UnboundedSender; #[cfg(any(test, target_os = "linux"))] use tokio::time::timeout; const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; +const PROXY_POD_NETWORK_ENFORCEMENT_MODE: &str = "proxy-pod"; const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; @@ -142,7 +145,9 @@ pub async fn run_sandbox( } } + let external_network_enforcement = external_network_enforcement_enabled(); let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); + let proxy_pod_network_enforcement = proxy_pod_network_enforcement_enabled(); let process_enforcement_mode = process_enforcement_mode(); let process_uses_sidecar_control = process_enabled && !network_enabled && sidecar_network_enforcement; @@ -164,6 +169,14 @@ pub async fn run_sandbox( } else { None }; + let supervisor_ready_addr = supervisor_ready_addr(); + if process_enabled + && !network_enabled + && proxy_pod_network_enforcement + && let Some(addr) = supervisor_ready_addr.as_deref() + { + wait_for_supervisor_ready_addr(addr).await?; + } // Extension credentials are owned by this supervisor and shared by every // gateway connection it opens, so the middleware registry's bearer slots @@ -388,7 +401,7 @@ pub async fn run_sandbox( // it via setns(). The RAII handle lives in this frame for the duration // of the sandbox. #[cfg(target_os = "linux")] - let netns = if network_enabled && !sidecar_network_enforcement { + let netns = if network_enabled && !external_network_enforcement { openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? } else { None @@ -549,11 +562,25 @@ pub async fn run_sandbox( None }; + let _gateway_forward = if network_enabled && proxy_pod_network_enforcement { + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return Err(miette::miette!( + "external network enforcement requires proxy network mode" + )); + } + let endpoint = openshell_endpoint_for_proxy.as_deref().ok_or_else(|| { + miette::miette!("proxy-pod network enforcement requires an OpenShell gateway endpoint") + })?; + Some(start_gateway_forward_from_env(endpoint).await?) + } else { + None + }; + #[cfg(target_os = "linux")] let sidecar_control_server = if network_enabled && sidecar_network_enforcement { if !matches!(policy.network.mode, NetworkMode::Proxy) { return Err(miette::miette!( - "sidecar network enforcement requires proxy network mode" + "external network enforcement requires proxy network mode" )); } let socket = sidecar_control_socket().ok_or_else(|| { @@ -622,9 +649,9 @@ pub async fn run_sandbox( } #[cfg(not(target_os = "linux"))] - if network_enabled && sidecar_network_enforcement { + if network_enabled && external_network_enforcement { return Err(miette::miette!( - "sidecar network enforcement is only supported on Linux" + "external network enforcement is only supported on Linux" )); } @@ -832,6 +859,8 @@ pub async fn run_sandbox( sidecar_bootstrap_ca_file_paths .clone() .or_else(sidecar_ca_file_paths) + } else if proxy_pod_network_enforcement { + sidecar_ca_file_paths() } else { None } @@ -1010,12 +1039,26 @@ fn sidecar_network_enforcement_enabled() -> bool { .is_ok_and(|value| value == SIDECAR_NETWORK_ENFORCEMENT_MODE) } +fn proxy_pod_network_enforcement_enabled() -> bool { + std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE) + .is_ok_and(|value| value == PROXY_POD_NETWORK_ENFORCEMENT_MODE) +} + +fn external_network_enforcement_enabled() -> bool { + std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE).is_ok_and(|value| { + matches!( + value.as_str(), + SIDECAR_NETWORK_ENFORCEMENT_MODE | PROXY_POD_NETWORK_ENFORCEMENT_MODE + ) + }) +} + fn process_enforcement_mode() -> ProcessEnforcementMode { match std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) .ok() .as_deref() { - Some("sidecar") => ProcessEnforcementMode::NetworkOnly, + Some("sidecar" | "proxy-pod") => ProcessEnforcementMode::NetworkOnly, _ => ProcessEnforcementMode::Full, } } @@ -1027,6 +1070,30 @@ fn sidecar_control_socket() -> Option { .map(std::path::PathBuf::from) } +fn supervisor_ready_addr() -> Option { + std::env::var(openshell_core::sandbox_env::SUPERVISOR_READY_ADDR) + .ok() + .filter(|value| !value.is_empty()) +} + +async fn wait_for_supervisor_ready_addr(addr: &str) -> Result<()> { + let deadline = tokio::time::Instant::now() + Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS); + loop { + match TcpStream::connect(addr).await { + Ok(_) => { + info!(addr, "Network supervisor TCP endpoint is ready"); + return Ok(()); + } + Err(err) if tokio::time::Instant::now() >= deadline => { + return Err(miette::miette!( + "timed out waiting for network supervisor TCP endpoint {addr}: {err}" + )); + } + Err(_) => tokio::time::sleep(Duration::from_millis(250)).await, + } + } +} + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] fn sidecar_expected_peer() -> Result { fn required_numeric_env(name: &str) -> Result { @@ -1288,6 +1355,100 @@ fn process_policy_for_topology( Ok(process_policy) } +struct GatewayForwardHandle { + task: tokio::task::JoinHandle<()>, +} + +impl Drop for GatewayForwardHandle { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn start_gateway_forward_from_env(endpoint: &str) -> Result { + let listen_addr = + std::env::var(openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR).map_err(|_| { + miette::miette!( + "{} is required for proxy-pod gateway forwarding", + openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR + ) + })?; + start_gateway_forward(&listen_addr, endpoint).await +} + +async fn start_gateway_forward(listen_addr: &str, endpoint: &str) -> Result { + let upstream = gateway_tcp_addr(endpoint)?; + let listener = TcpListener::bind(listen_addr).await.into_diagnostic()?; + info!( + listen_addr, + upstream, "Gateway TCP forward started for proxy-pod topology" + ); + + let task = tokio::spawn(async move { + loop { + let (mut inbound, peer) = match listener.accept().await { + Ok(accepted) => accepted, + Err(e) => { + warn!(error = %e, "Gateway forward accept failed"); + continue; + } + }; + let upstream = upstream.clone(); + tokio::spawn(async move { + let mut outbound = match TcpStream::connect(&upstream).await { + Ok(stream) => stream, + Err(e) => { + warn!(peer = %peer, upstream, error = %e, "Gateway forward connect failed"); + return; + } + }; + if let Err(e) = copy_bidirectional(&mut inbound, &mut outbound).await { + debug!(peer = %peer, error = %e, "Gateway forward connection closed with error"); + } + }); + } + }); + + Ok(GatewayForwardHandle { task }) +} + +fn gateway_tcp_addr(endpoint: &str) -> Result { + let (scheme, rest) = endpoint + .split_once("://") + .ok_or_else(|| miette::miette!("gateway endpoint must include a URL scheme"))?; + let default_port = match scheme { + "http" => 80, + "https" => 443, + other => { + return Err(miette::miette!( + "unsupported gateway endpoint scheme '{other}' for proxy-pod forwarding" + )); + } + }; + let authority = rest.split('/').next().unwrap_or(rest); + if authority.is_empty() { + return Err(miette::miette!("gateway endpoint is missing a host")); + } + if authority.starts_with('[') { + let closing = authority + .find(']') + .ok_or_else(|| miette::miette!("invalid bracketed IPv6 gateway endpoint"))?; + let host = &authority[..=closing]; + let port = authority[closing + 1..] + .strip_prefix(':') + .and_then(|value| value.parse::().ok()) + .unwrap_or(default_port); + return Ok(format!("{host}:{port}")); + } + let (host, port) = match authority.rsplit_once(':') { + Some((host, port)) if !host.is_empty() => { + (host, port.parse::().unwrap_or(default_port)) + } + _ => (authority, default_port), + }; + Ok(format!("{host}:{port}")) +} + /// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. async fn flush_proposals_to_gateway( endpoint: &str, diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 64e77ef600..2a9b77ee02 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -186,8 +186,9 @@ struct Args { #[arg(long, default_value = DEFAULT_MODE)] mode: Mode, - /// UID that the long-running Kubernetes network sidecar will run as. - /// `--mode=network-init` installs nftables rules that exempt this UID. + /// UID that the long-running Kubernetes network proxy will run as. + /// In sidecar topology, `--mode=network-init` installs nftables rules + /// that exempt this UID. #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] proxy_uid: u32, @@ -537,10 +538,10 @@ fn main() -> Result<()> { let args = Args::parse(); if args.mode.network_init { - let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); + let proxy_group_id = args.proxy_gid.unwrap_or(args.proxy_uid); return run_network_init( args.proxy_uid, - proxy_gid, + proxy_group_id, &args.sidecar_state_dir, &args.sidecar_tls_dir, ); diff --git a/crates/openshell-server/src/auth/k8s_sa.rs b/crates/openshell-server/src/auth/k8s_sa.rs index 131dbaba47..30afaee521 100644 --- a/crates/openshell-server/src/auth/k8s_sa.rs +++ b/crates/openshell-server/src/auth/k8s_sa.rs @@ -5,8 +5,9 @@ //! //! Path-scoped to `IssueSandboxToken`. Validates a projected SA token //! presented by a sandbox pod, reads the pod's `openshell.io/sandbox-id` -//! annotation, verifies the pod is controlled by the corresponding Sandbox CR, -//! and returns a [`Principal::Sandbox`] with +//! annotation, verifies the pod is controlled by the corresponding Sandbox CR +//! either directly or through a supervisor Deployment controller chain, and +//! returns a [`Principal::Sandbox`] with //! [`SandboxIdentitySource::K8sServiceAccount`]. The `IssueSandboxToken` handler //! then mints a gateway-signed JWT for that sandbox id; subsequent gRPC calls //! from the supervisor use the gateway-minted JWT validated by @@ -19,10 +20,11 @@ use super::authenticator::Authenticator; use super::principal::{Principal, SandboxIdentitySource, SandboxPrincipal}; use async_trait::async_trait; use k8s_openapi::api::{ + apps::v1::{Deployment, ReplicaSet}, authentication::v1::{TokenReview, TokenReviewSpec, TokenReviewStatus, UserInfo}, core::v1::Pod, }; -use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, OwnerReference}; use kube::Error as KubeError; use kube::api::{Api, ApiResource, PostParams}; use kube::core::{DynamicObject, gvk::GroupVersionKind}; @@ -46,7 +48,10 @@ const SANDBOX_API_VERSION_V1BETA1: &str = "v1beta1"; const SANDBOX_API_VERSION_V1ALPHA1: &str = "v1alpha1"; const SANDBOX_API_VERSION_FULL_V1BETA1: &str = "agents.x-k8s.io/v1beta1"; const SANDBOX_API_VERSION_FULL_V1ALPHA1: &str = "agents.x-k8s.io/v1alpha1"; +const APPS_API_VERSION_FULL_V1: &str = "apps/v1"; const SANDBOX_KIND: &str = "Sandbox"; +const REPLICA_SET_KIND: &str = "ReplicaSet"; +const DEPLOYMENT_KIND: &str = "Deployment"; const SANDBOX_ID_LABEL: &str = "openshell.ai/sandbox-id"; const POD_NAME_EXTRA: &str = "authentication.kubernetes.io/pod-name"; const POD_UID_EXTRA: &str = "authentication.kubernetes.io/pod-uid"; @@ -173,6 +178,14 @@ struct SandboxOwnerReference { uid: String, } +#[derive(Debug, Clone, PartialEq, Eq)] +struct ControllerOwnerReference { + api_version: String, + kind: String, + name: String, + uid: String, +} + /// Resolver backed by the apiserver's `TokenReview` API and `kube::Client` /// for the per-pod annotation lookup. pub struct LiveK8sResolver { @@ -233,6 +246,139 @@ impl LiveK8sResolver { Ok(None) } + + fn replica_sets_api(&self, namespace: &str) -> Api { + Api::namespaced(self.client.clone(), namespace) + } + + fn deployments_api(&self, namespace: &str) -> Api { + Api::namespaced(self.client.clone(), namespace) + } + + async fn sandbox_owner_for_pod( + &self, + pod: &Pod, + namespace: &str, + pod_name: &str, + ) -> Result { + match direct_sandbox_owner_reference(pod) { + Ok(owner) => Ok(owner), + Err(err) => { + let Some(controller) = controller_owner_reference( + pod.metadata.owner_references.as_deref().unwrap_or_default(), + ) else { + return Err(err); + }; + if controller.api_version != APPS_API_VERSION_FULL_V1 + || controller.kind != REPLICA_SET_KIND + { + return Err(err); + } + self.sandbox_owner_for_replica_set_controller(&controller, namespace, pod_name) + .await + } + } + } + + async fn sandbox_owner_for_replica_set_controller( + &self, + replica_set_owner: &ControllerOwnerReference, + namespace: &str, + pod_name: &str, + ) -> Result { + let replica_set = self + .replica_sets_api(namespace) + .get_opt(&replica_set_owner.name) + .await + .map_err(|e| { + warn!( + pod = %pod_name, + replica_set = %replica_set_owner.name, + error = %e, + "failed to fetch ReplicaSet for pod identity validation" + ); + Status::internal(format!("replicaset GET failed: {e}")) + })? + .ok_or_else(|| { + warn!( + pod = %pod_name, + replica_set = %replica_set_owner.name, + "pod controller ReplicaSet was not found" + ); + Status::permission_denied("pod controller ReplicaSet not found") + })?; + validate_object_uid( + replica_set.metadata.uid.as_deref().unwrap_or_default(), + &replica_set_owner.uid, + "pod controller ReplicaSet UID mismatch", + )?; + + let deployment_owner = controller_owner_reference( + replica_set + .metadata + .owner_references + .as_deref() + .unwrap_or_default(), + ) + .ok_or_else(|| { + warn!( + pod = %pod_name, + replica_set = %replica_set_owner.name, + "ReplicaSet has no controlling Deployment ownerReference" + ); + Status::permission_denied("ReplicaSet is not controlled by a Deployment") + })?; + if deployment_owner.api_version != APPS_API_VERSION_FULL_V1 + || deployment_owner.kind != DEPLOYMENT_KIND + { + warn!( + pod = %pod_name, + replica_set = %replica_set_owner.name, + owner_api_version = %deployment_owner.api_version, + owner_kind = %deployment_owner.kind, + "ReplicaSet controller is not an apps/v1 Deployment" + ); + return Err(Status::permission_denied( + "ReplicaSet is not controlled by a Deployment", + )); + } + + let deployment = self + .deployments_api(namespace) + .get_opt(&deployment_owner.name) + .await + .map_err(|e| { + warn!( + pod = %pod_name, + deployment = %deployment_owner.name, + error = %e, + "failed to fetch Deployment for pod identity validation" + ); + Status::internal(format!("deployment GET failed: {e}")) + })? + .ok_or_else(|| { + warn!( + pod = %pod_name, + deployment = %deployment_owner.name, + "ReplicaSet controller Deployment was not found" + ); + Status::permission_denied("ReplicaSet controller Deployment not found") + })?; + validate_object_uid( + deployment.metadata.uid.as_deref().unwrap_or_default(), + &deployment_owner.uid, + "ReplicaSet controller Deployment UID mismatch", + )?; + + sandbox_owner_reference_from_owner_refs( + deployment + .metadata + .owner_references + .as_deref() + .unwrap_or_default(), + "Deployment", + ) + } } #[async_trait] @@ -308,7 +454,9 @@ impl K8sIdentityResolver for LiveK8sResolver { let sandbox_id = pod_sandbox_id(&pod)?; - let owner = sandbox_owner_reference(&pod)?; + let owner = self + .sandbox_owner_for_pod(&pod, &identity.namespace, &identity.pod_name) + .await?; let sandbox_cr = self .get_sandbox_cr_for_owner(&identity.namespace, &owner) .await @@ -455,8 +603,18 @@ fn pod_sandbox_id(pod: &Pod) -> Result { } #[allow(clippy::result_large_err)] -fn sandbox_owner_reference(pod: &Pod) -> Result { - let owner_refs = pod.metadata.owner_references.as_deref().unwrap_or_default(); +fn direct_sandbox_owner_reference(pod: &Pod) -> Result { + sandbox_owner_reference_from_owner_refs( + pod.metadata.owner_references.as_deref().unwrap_or_default(), + "pod", + ) +} + +#[allow(clippy::result_large_err)] +fn sandbox_owner_reference_from_owner_refs( + owner_refs: &[OwnerReference], + object_kind: &str, +) -> Result { let mut sandbox_refs = owner_refs .iter() .filter(|owner| is_supported_sandbox_owner_reference(owner)); @@ -473,27 +631,28 @@ fn sandbox_owner_reference(pod: &Pod) -> Result { SANDBOX_API_VERSION_FULL_V1BETA1, SANDBOX_API_VERSION_FULL_V1ALPHA1, ], - "pod Sandbox ownerReference uses unsupported apiVersion" + object_kind = %object_kind, + "Sandbox ownerReference uses unsupported apiVersion" ); } - return Err(Status::permission_denied( - "pod is not controlled by an OpenShell Sandbox", - )); + return Err(Status::permission_denied(format!( + "{object_kind} is not controlled by an OpenShell Sandbox" + ))); }; if sandbox_refs.next().is_some() { - return Err(Status::permission_denied( - "pod has multiple OpenShell Sandbox owners", - )); + return Err(Status::permission_denied(format!( + "{object_kind} has multiple OpenShell Sandbox owners" + ))); } if owner.controller != Some(true) { - return Err(Status::permission_denied( - "pod Sandbox ownerReference is not controlling", - )); + return Err(Status::permission_denied(format!( + "{object_kind} Sandbox ownerReference is not controlling" + ))); } if owner.name.is_empty() || owner.uid.is_empty() { - return Err(Status::permission_denied( - "pod Sandbox ownerReference is incomplete", - )); + return Err(Status::permission_denied(format!( + "{object_kind} Sandbox ownerReference is incomplete" + ))); } Ok(SandboxOwnerReference { api_version: owner.api_version.clone(), @@ -502,9 +661,32 @@ fn sandbox_owner_reference(pod: &Pod) -> Result { }) } -fn is_supported_sandbox_owner_reference( - owner: &k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference, -) -> bool { +fn controller_owner_reference(owner_refs: &[OwnerReference]) -> Option { + let owner = owner_refs + .iter() + .find(|owner| owner.controller == Some(true))?; + Some(ControllerOwnerReference { + api_version: owner.api_version.clone(), + kind: owner.kind.clone(), + name: owner.name.clone(), + uid: owner.uid.clone(), + }) +} + +#[allow(clippy::result_large_err)] +fn validate_object_uid(actual_uid: &str, expected_uid: &str, message: &str) -> Result<(), Status> { + if actual_uid != expected_uid { + warn!( + expected_uid = %expected_uid, + actual_uid = %actual_uid, + %message + ); + return Err(Status::permission_denied(message.to_string())); + } + Ok(()) +} + +fn is_supported_sandbox_owner_reference(owner: &OwnerReference) -> bool { owner.kind == SANDBOX_KIND && matches!( owner.api_version.as_str(), @@ -678,6 +860,17 @@ mod tests { } } + fn app_controller_owner(kind: &str, name: &str, uid: &str) -> OwnerReference { + OwnerReference { + api_version: APPS_API_VERSION_FULL_V1.to_string(), + block_owner_deletion: None, + controller: Some(true), + kind: kind.to_string(), + name: name.to_string(), + uid: uid.to_string(), + } + } + fn pod_with_owner_refs(owner_references: Vec) -> Pod { Pod { metadata: ObjectMeta { @@ -898,7 +1091,7 @@ mod tests { fn sandbox_owner_reference_extracts_controlling_sandbox_owner() { let pod = pod_with_owner_refs(vec![sandbox_owner("sandbox-a", "cr-uid-a")]); - let owner = sandbox_owner_reference(&pod).expect("expected Sandbox owner"); + let owner = direct_sandbox_owner_reference(&pod).expect("expected Sandbox owner"); assert_eq!( owner, @@ -918,7 +1111,7 @@ mod tests { "cr-uid-a", )]); - let owner = sandbox_owner_reference(&pod).expect("expected v1alpha1 Sandbox owner"); + let owner = direct_sandbox_owner_reference(&pod).expect("expected v1alpha1 Sandbox owner"); assert_eq!( owner, @@ -934,7 +1127,7 @@ mod tests { fn sandbox_owner_reference_rejects_missing_owner() { let pod = pod_with_owner_refs(vec![]); - let err = sandbox_owner_reference(&pod).expect_err("missing owner must fail"); + let err = direct_sandbox_owner_reference(&pod).expect_err("missing owner must fail"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } @@ -947,8 +1140,8 @@ mod tests { "cr-uid-a", )]); - let err = - sandbox_owner_reference(&pod).expect_err("unsupported apiVersion must fail closed"); + let err = direct_sandbox_owner_reference(&pod) + .expect_err("unsupported apiVersion must fail closed"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } @@ -959,7 +1152,7 @@ mod tests { owner.controller = Some(false); let pod = pod_with_owner_refs(vec![owner]); - let err = sandbox_owner_reference(&pod).expect_err("non-controller owner must fail"); + let err = direct_sandbox_owner_reference(&pod).expect_err("non-controller owner must fail"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } @@ -971,11 +1164,50 @@ mod tests { sandbox_owner("sandbox-b", "cr-uid-b"), ]); - let err = sandbox_owner_reference(&pod).expect_err("multiple owners must fail"); + let err = direct_sandbox_owner_reference(&pod).expect_err("multiple owners must fail"); assert_eq!(err.code(), tonic::Code::PermissionDenied); } + #[test] + fn controller_owner_reference_extracts_controlling_apps_owner() { + let pod = pod_with_owner_refs(vec![app_controller_owner( + REPLICA_SET_KIND, + "supervisor-rs", + "rs-uid", + )]); + + let owner = controller_owner_reference(pod.metadata.owner_references.as_deref().unwrap()) + .expect("expected controller owner"); + + assert_eq!( + owner, + ControllerOwnerReference { + api_version: APPS_API_VERSION_FULL_V1.to_string(), + kind: REPLICA_SET_KIND.to_string(), + name: "supervisor-rs".to_string(), + uid: "rs-uid".to_string(), + } + ); + } + + #[test] + fn sandbox_owner_reference_from_deployment_requires_controlling_sandbox_owner() { + let deployment_owner_refs = vec![sandbox_owner("sandbox-a", "cr-uid-a")]; + + let owner = sandbox_owner_reference_from_owner_refs(&deployment_owner_refs, "Deployment") + .expect("expected Deployment Sandbox owner"); + + assert_eq!( + owner, + SandboxOwnerReference { + api_version: SANDBOX_API_VERSION_FULL_V1BETA1.to_string(), + name: "sandbox-a".to_string(), + uid: "cr-uid-a".to_string(), + } + ); + } + #[test] fn validate_sandbox_owner_reference_requires_matching_cr_uid_and_label() { let owner = SandboxOwnerReference { diff --git a/crates/openshell-supervisor-network/src/l7/tls.rs b/crates/openshell-supervisor-network/src/l7/tls.rs index 2275a60d34..0de47e545a 100644 --- a/crates/openshell-supervisor-network/src/l7/tls.rs +++ b/crates/openshell-supervisor-network/src/l7/tls.rs @@ -63,6 +63,28 @@ impl SandboxCa { }) } + /// Load an existing CA certificate and private key from PEM. + pub fn from_pem(ca_cert_pem: &str, ca_key_pem: &str) -> Result { + let ca_key = KeyPair::from_pem(ca_key_pem).into_diagnostic()?; + let ca_cert = CertificateParams::from_ca_cert_pem(ca_cert_pem) + .into_diagnostic()? + .self_signed(&ca_key) + .into_diagnostic()?; + + Ok(Self { + ca_cert, + ca_key, + ca_cert_pem: ca_cert_pem.to_string(), + }) + } + + /// Load an existing CA certificate and private key from files. + pub fn from_files(cert_path: &Path, key_path: &Path) -> Result { + let ca_cert_pem = std::fs::read_to_string(cert_path).into_diagnostic()?; + let ca_key_pem = std::fs::read_to_string(key_path).into_diagnostic()?; + Self::from_pem(&ca_cert_pem, &ca_key_pem) + } + /// Returns the CA certificate in PEM format. pub fn cert_pem(&self) -> &str { &self.ca_cert_pem @@ -559,4 +581,18 @@ mod tests { "bundle should contain at least one cert", ); } + + #[test] + fn sandbox_ca_loads_from_pem() { + let ca = SandboxCa::generate().unwrap(); + let key_pem = ca.ca_key.serialize_pem(); + let loaded = SandboxCa::from_pem(ca.cert_pem(), &key_pem).unwrap(); + + assert_eq!(loaded.cert_pem(), ca.cert_pem()); + assert!( + CertCache::new(loaded) + .get_or_generate("example.com") + .is_ok() + ); + } } diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index a9170ceee7..5f2a581918 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -161,6 +161,38 @@ pub struct Networking { _transparent_tcp: Option, } +fn sandbox_ca_for_proxy() -> Result { + let cert_path = std::env::var(openshell_core::sandbox_env::PROXY_CA_CERT_PATH).ok(); + let key_path = std::env::var(openshell_core::sandbox_env::PROXY_CA_KEY_PATH).ok(); + match (cert_path, key_path) { + (Some(cert_path), Some(key_path)) => SandboxCa::from_files( + std::path::Path::new(&cert_path), + std::path::Path::new(&key_path), + ), + (None, None) => SandboxCa::generate(), + _ => Err(miette::miette!( + "{} and {} must be set together", + openshell_core::sandbox_env::PROXY_CA_CERT_PATH, + openshell_core::sandbox_env::PROXY_CA_KEY_PATH + )), + } +} + +fn explicit_proxy_bind_addr() -> Result> { + let Some(value) = std::env::var(openshell_core::sandbox_env::PROXY_BIND_ADDR) + .ok() + .filter(|value| !value.trim().is_empty()) + else { + return Ok(None); + }; + value.parse::().map(Some).map_err(|err| { + miette::miette!( + "invalid {} value {value:?}: {err}", + openshell_core::sandbox_env::PROXY_BIND_ADDR + ) + }) +} + /// Set up the networking stack: ephemeral CA + TLS state, proxy server, /// and the SSH-side proxy URL / netns FD. /// @@ -313,10 +345,10 @@ pub async fn run_networking( // the proxy, so it's owned here. let identity_cache = opa_engine.map(|_| Arc::new(BinaryIdentityCache::new())); - // Generate ephemeral CA and TLS state for HTTPS L7 inspection. - // The CA cert is written to disk so sandbox processes can trust it. + // Generate or load a CA and TLS state for HTTPS L7 inspection. The CA cert + // is written to disk so sandbox processes can trust it. let (tls_state, ca_file_paths) = if matches!(policy.network.mode, NetworkMode::Proxy) { - match SandboxCa::generate() { + match sandbox_ca_for_proxy() { Ok(ca) => { let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) .unwrap_or_else(|_| openshell_core::container_paths::TLS_ROOT.to_string()); @@ -336,7 +368,7 @@ pub async fn run_networking( .severity(SeverityId::Informational) .status(StatusId::Success) .state(StateId::Enabled, "enabled") - .message("TLS termination enabled: ephemeral CA generated") + .message("TLS termination enabled") .build() ); (Some(state), Some(paths)) @@ -371,7 +403,7 @@ pub async fn run_networking( .status(StatusId::Failure) .state(StateId::Disabled, "disabled") .message(format!( - "Failed to generate ephemeral CA, TLS termination disabled: {e}" + "Failed to initialize proxy CA, TLS termination disabled: {e}" )) .build() ); @@ -400,9 +432,11 @@ pub async fn run_networking( // originating inside the namespace can reach the proxy. Otherwise the // proxy falls back to the policy-declared http_addr (loopback in // tests, etc.). - let bind_addr = proxy_bind_ip.map(|ip| { - let port = proxy_policy.http_addr.map_or(3128, |addr| addr.port()); - SocketAddr::new(ip, port) + let bind_addr = explicit_proxy_bind_addr()?.or_else(|| { + proxy_bind_ip.map(|ip| { + let port = proxy_policy.http_addr.map_or(3128, |addr| addr.port()); + SocketAddr::new(ip, port) + }) }); // Build inference context for local routing of intercepted inference calls. diff --git a/crates/openshell-supervisor-process/Cargo.toml b/crates/openshell-supervisor-process/Cargo.toml index 2e2120f1d0..3238770cac 100644 --- a/crates/openshell-supervisor-process/Cargo.toml +++ b/crates/openshell-supervisor-process/Cargo.toml @@ -45,6 +45,7 @@ socket2 = { workspace = true } tempfile = "3" [dev-dependencies] +temp-env = "0.3" tempfile = "3" [lints] diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 2b4ea554ed..6c6ddaa97c 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -641,7 +641,7 @@ pub fn create_netns_for_proxy( /// Install pod-network bypass enforcement for Kubernetes sidecar topology. /// /// This runs in the current network namespace, not in a per-workload netns. -/// The rules allow loopback and the sidecar proxy UID, then reject direct +/// The rules allow loopback and the proxy UID, then reject direct /// TCP/UDP egress from other UIDs so traffic must use the sidecar's local /// proxy. /// diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index b61a51c5b2..ddfd65a875 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -166,6 +166,10 @@ const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ openshell_core::sandbox_env::TLS_KEY, openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::PROXY_URL, + openshell_core::sandbox_env::PROXY_BIND_ADDR, + openshell_core::sandbox_env::PROXY_CA_CERT_PATH, + openshell_core::sandbox_env::PROXY_CA_KEY_PATH, ]; pub fn is_supervisor_only_env_var(key: &str) -> bool { @@ -248,6 +252,35 @@ fn configured_user_environment() -> HashMap { .unwrap_or_default() } +fn configured_proxy_url( + policy: &SandboxPolicy, + netns_proxy_enabled: bool, +) -> Result> { + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return Ok(None); + } + + if let Ok(proxy_url) = std::env::var(openshell_core::sandbox_env::PROXY_URL) { + let trimmed = proxy_url.trim(); + if !trimmed.is_empty() { + return Ok(Some(trimmed.to_string())); + } + } + + let proxy = policy.network.proxy.as_ref().ok_or_else(|| { + miette::miette!("Network mode is set to proxy but no proxy configuration was provided") + })?; + + if netns_proxy_enabled { + let port = proxy.http_addr.map_or(3128, |addr| addr.port()); + return Ok(Some(format!("http://10.200.0.1:{port}"))); + } + + Ok(proxy + .http_addr + .map(|http_addr| format!("http://{http_addr}"))) +} + #[cfg(unix)] pub fn harden_child_process() -> Result<()> { use rustix::process::{Resource, Rlimit, setrlimit}; @@ -761,27 +794,11 @@ impl ProcessHandle { cmd.current_dir(dir); } - if matches!(policy.network.mode, NetworkMode::Proxy) { - let proxy = policy.network.proxy.as_ref().ok_or_else(|| { - miette::miette!( - "Network mode is set to proxy but no proxy configuration was provided" - ) - })?; - // When using network namespace, set proxy URL to the veth host IP - if netns_fd.is_some() { - // The proxy is on 10.200.0.1:3128 (or configured port) - let port = proxy.http_addr.map_or(3128, |addr| addr.port()); - let proxy_url = format!("http://10.200.0.1:{port}"); - // Both uppercase and lowercase variants: curl/wget use uppercase, - // gRPC C-core (libgrpc) checks lowercase http_proxy/https_proxy. - for (key, value) in child_env::proxy_env_vars(&proxy_url) { - cmd.env(key, value); - } - } else if let Some(http_addr) = proxy.http_addr { - let proxy_url = format!("http://{http_addr}"); - for (key, value) in child_env::proxy_env_vars(&proxy_url) { - cmd.env(key, value); - } + if let Some(proxy_url) = configured_proxy_url(policy, netns_fd.is_some())? { + // Both uppercase and lowercase variants: curl/wget use uppercase, + // gRPC C-core (libgrpc) checks lowercase http_proxy/https_proxy. + for (key, value) in child_env::proxy_env_vars(&proxy_url) { + cmd.env(key, value); } } @@ -960,17 +977,9 @@ impl ProcessHandle { cmd.current_dir(dir); } - if matches!(policy.network.mode, NetworkMode::Proxy) { - let proxy = policy.network.proxy.as_ref().ok_or_else(|| { - miette::miette!( - "Network mode is set to proxy but no proxy configuration was provided" - ) - })?; - if let Some(http_addr) = proxy.http_addr { - let proxy_url = format!("http://{http_addr}"); - for (key, value) in child_env::proxy_env_vars(&proxy_url) { - cmd.env(key, value); - } + if let Some(proxy_url) = configured_proxy_url(policy, false)? { + for (key, value) in child_env::proxy_env_vars(&proxy_url) { + cmd.env(key, value); } } diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index 34b0001110..939a7b3f4f 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -601,6 +601,13 @@ fn ssh_proxy_url_for_policy( return None; } + if let Ok(proxy_url) = std::env::var(openshell_core::sandbox_env::PROXY_URL) { + let trimmed = proxy_url.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + let proxy = policy.network.proxy.as_ref()?; if let Some(host) = netns_proxy_host { let port = proxy.http_addr.map_or(3128, |addr| addr.port()); @@ -669,6 +676,8 @@ mod tests { FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, }; + static PROXY_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + fn policy(mode: NetworkMode, http_addr: Option) -> SandboxPolicy { SandboxPolicy { version: 1, @@ -684,30 +693,56 @@ mod tests { } } + fn with_proxy_url(proxy_url: Option<&str>, test: F) -> T + where + F: FnOnce() -> T, + { + let _guard = PROXY_ENV_LOCK.lock().expect("proxy env lock poisoned"); + temp_env::with_var(openshell_core::sandbox_env::PROXY_URL, proxy_url, test) + } + #[test] fn ssh_proxy_url_uses_policy_addr_without_netns() { - let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 3128).into())); + with_proxy_url(None, || { + let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 3128).into())); - assert_eq!( - ssh_proxy_url_for_policy(&policy, None).as_deref(), - Some("http://127.0.0.1:3128") - ); + assert_eq!( + ssh_proxy_url_for_policy(&policy, None).as_deref(), + Some("http://127.0.0.1:3128") + ); + }); } #[test] fn ssh_proxy_url_prefers_netns_host_with_policy_port() { - let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 8080).into())); + with_proxy_url(None, || { + let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 8080).into())); - assert_eq!( - ssh_proxy_url_for_policy(&policy, Some([10, 200, 0, 1].into())).as_deref(), - Some("http://10.200.0.1:8080") - ); + assert_eq!( + ssh_proxy_url_for_policy(&policy, Some([10, 200, 0, 1].into())).as_deref(), + Some("http://10.200.0.1:8080") + ); + }); } #[test] fn ssh_proxy_url_skips_non_proxy_mode() { - let policy = policy(NetworkMode::Allow, Some(([127, 0, 0, 1], 3128).into())); + with_proxy_url(None, || { + let policy = policy(NetworkMode::Allow, Some(([127, 0, 0, 1], 3128).into())); + + assert_eq!(ssh_proxy_url_for_policy(&policy, None), None); + }); + } + + #[test] + fn ssh_proxy_url_prefers_env_override() { + with_proxy_url(Some("http://openshell-supervisor.default.svc:3128"), || { + let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 8080).into())); - assert_eq!(ssh_proxy_url_for_policy(&policy, None), None); + assert_eq!( + ssh_proxy_url_for_policy(&policy, Some([10, 200, 0, 1].into())).as_deref(), + Some("http://openshell-supervisor.default.svc:3128") + ); + }); } } diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 817900e2d2..d120122bde 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -284,10 +284,11 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | +| supervisor.proxyPod.proxyUid | int | `1337` | UID for the network supervisor in proxy-pod topology. The configured UID must not match the sandbox UID. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | | supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. | | supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | -| supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. | +| supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. "proxy-pod" runs network enforcement in a separate supervisor Deployment and restricts the agent pod to that supervisor through NetworkPolicy. | | tolerations | list | `[]` | Tolerations for the gateway pod. | | upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. | | upstreamProxy.authAllowInsecure | bool | `false` | Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. | diff --git a/deploy/helm/openshell/ci/values-proxy-pod.yaml b/deploy/helm/openshell/ci/values-proxy-pod.yaml new file mode 100644 index 0000000000..b7cb533fd7 --- /dev/null +++ b/deploy/helm/openshell/ci/values-proxy-pod.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# CI/dev overlay for exercising the Kubernetes proxy-pod topology. +# +# This topology relies on Kubernetes NetworkPolicy enforcement: the agent pod is +# isolated to its paired supervisor pod plus DNS. The local k3s/k3d workflow +# must therefore run with the k3s network policy controller enabled, or with a +# custom policy-enforcing CNI installed before deploying this profile. +# +# Merge after values.yaml and ci/values-skaffold.yaml: +# helm install ... -f values.yaml -f ci/values-skaffold.yaml -f ci/values-proxy-pod.yaml +# +# Or set: +# OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-proxy-pod.yaml +# before running `mise run e2e:kubernetes`. +supervisor: + topology: proxy-pod diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index ce32c72132..153961dfcb 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -121,6 +121,11 @@ deploy: #- ci/values-spire.yaml # To exercise the Kubernetes supervisor sidecar topology: #- ci/values-sidecar.yaml + # To exercise proxy-pod topology, use the proxy-pod Skaffold profile + # against a cluster with NetworkPolicy enforcement enabled. Stock k3s + # includes its embedded network policy controller; if you replace the + # CNI, install a policy-enforcing CNI before deploying this profile. + #- ci/values-proxy-pod.yaml # To test multi-replica external PostgreSQL behavior: #- ci/values-high-availability.yaml setValueTemplates: @@ -153,3 +158,8 @@ profiles: - op: add path: /deploy/helm/releases/0/valuesFiles/- value: ci/values-credential-driver-vault.yaml + - name: proxy-pod + patches: + - op: add + path: /deploy/helm/releases/0/valuesFiles/- + value: ci/values-proxy-pod.yaml diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 9d24dbd917..23fbfa5229 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -206,6 +206,9 @@ data: proxy_uid = {{ .Values.supervisor.sidecar.proxyUid | default 1337 }} process_binary_aware_network_policy = {{ .Values.supervisor.sidecar.processBinaryAwareNetworkPolicy }} + [openshell.drivers.kubernetes.proxy_pod] + proxy_uid = {{ .Values.supervisor.proxyPod.proxyUid | default 1337 }} + {{- if not $credentialDrivers }} [openshell.gateway.credential_storage] diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index af80989072..41ec08942b 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -36,11 +36,59 @@ rules: # returned pod name and UID to the pod's `openshell.io/sandbox-id` # annotation. patch is intentionally NOT granted — the annotation is set # once at pod create and must remain immutable for the lifetime of the - # sandbox. + # sandbox. create/delete/list/watch are intentionally not granted; the Agent + # Sandbox controller creates agent pods, and proxy-pod supervisors are + # managed through per-sandbox Deployments. - apiGroups: - "" resources: - pods verbs: - get + {{- if eq (.Values.supervisor.topology | default "combined") "proxy-pod" }} + # Proxy-pod topology creates one supervisor Deployment, one supervisor + # Service, and one CA Secret per sandbox. All are owner-referenced to the + # Sandbox CR for garbage collection. The gateway also reads the generated + # ReplicaSet during K8s ServiceAccount bootstrap to verify the supervisor + # pod's Pod -> ReplicaSet -> Deployment -> Sandbox owner chain. These + # permissions are only rendered when the Kubernetes driver is configured for + # proxy-pod topology. + - apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - watch + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - apiGroups: + - "" + resources: + - services + - secrets + verbs: + - create + - delete + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - watch + {{- end }} {{- end }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index afacd01eb4..fd100718f4 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -190,6 +190,18 @@ tests: path: data["gateway.toml"] pattern: 'supervisor[_]topology\s*=' + - it: renders proxy-pod supervisor topology under [openshell.drivers.kubernetes] + template: templates/gateway-config.yaml + set: + supervisor.topology: proxy-pod + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?topology\s*=\s*"proxy-pod"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'supervisor[_]topology\s*=' + - it: renders proxy uid under [openshell.drivers.kubernetes.sidecar] template: templates/gateway-config.yaml set: @@ -199,6 +211,15 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?proxy_uid\s*=\s*2200' + - it: renders proxy uid under [openshell.drivers.kubernetes.proxy_pod] + template: templates/gateway-config.yaml + set: + supervisor.proxyPod.proxyUid: 2300 + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.proxy_pod\].*?proxy_uid\s*=\s*2300' + - it: renders process binary aware network policy under [openshell.drivers.kubernetes.sidecar] template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index ee89fce53d..5be3f1d9db 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -57,6 +57,139 @@ tests: path: metadata.namespace value: other-ns + - it: grants only pod get for sandbox token bootstrap + template: templates/role.yaml + asserts: + - contains: + path: rules + content: + apiGroups: + - "" + resources: + - pods + verbs: + - get + + - it: grants sandbox RBAC for proxy-pod supervisor Deployments + template: templates/role.yaml + set: + supervisor.topology: proxy-pod + asserts: + - contains: + path: rules + content: + apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - watch + + - it: grants ReplicaSet get for proxy-pod supervisor token bootstrap + template: templates/role.yaml + set: + supervisor.topology: proxy-pod + asserts: + - contains: + path: rules + content: + apiGroups: + - apps + resources: + - replicasets + verbs: + - get + + - it: grants proxy-pod Service Secret and NetworkPolicy RBAC only in proxy-pod mode + template: templates/role.yaml + set: + supervisor.topology: proxy-pod + asserts: + - contains: + path: rules + content: + apiGroups: + - "" + resources: + - services + - secrets + verbs: + - create + - delete + - get + - list + - watch + - contains: + path: rules + content: + apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - watch + + - it: omits proxy-pod RBAC in the default combined topology + template: templates/role.yaml + asserts: + - notContains: + path: rules + content: + apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - watch + - notContains: + path: rules + content: + apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - notContains: + path: rules + content: + apiGroups: + - "" + resources: + - services + - secrets + verbs: + - create + - delete + - get + - list + - watch + - notContains: + path: rules + content: + apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - watch + - it: uses explicit sandboxNamespace for sandbox RoleBinding template: templates/rolebinding.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index b4b14ef9d3..324a6dad78 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -48,6 +48,8 @@ supervisor: # "combined" runs the current single supervisor container in the agent pod. # "sidecar" runs network enforcement in a dedicated sidecar and the process # supervisor as a low-capability wrapper in the agent container. + # "proxy-pod" runs network enforcement in a separate supervisor Deployment and + # restricts the agent pod to that supervisor through NetworkPolicy. topology: "combined" sidecar: # -- UID for relaxed long-running network sidecars in sidecar topology. @@ -61,6 +63,10 @@ supervisor: # inspection capabilities, and enforces endpoint/L7 policy without matching # policy.binaries. processBinaryAwareNetworkPolicy: true + proxyPod: + # -- UID for the network supervisor in proxy-pod topology. The configured + # UID must not match the sandbox UID. + proxyUid: 1337 # -- Operator-owned corporate forward proxy for policy-approved TLS egress # from Kubernetes sandboxes. The workload cannot select or override it. diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index 221f935eb6..21cd0828e9 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -177,6 +177,7 @@ The most commonly changed values are: | `supervisor.topology` | Sandbox pod topology. Refer to [Topology](/kubernetes/topology). | | `supervisor.sidecar.proxyUid` | Non-root UID used when sidecar process/binary-aware network policy is disabled. The default binary-aware sidecar runs as UID 0 instead. The configured UID must not match the sandbox UID. | | `upstreamProxy` | Operator-owned corporate HTTP forward proxy for policy-approved TLS egress. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). | +| `supervisor.proxyPod.proxyUid` | Non-root UID used by the proxy-pod network supervisor. The UID must not match the sandbox UID. | Use a values file for repeatable deployments: @@ -260,6 +261,10 @@ The namespaced Role covers sandbox lifecycle and identity: | `agents.x-k8s.io` | `sandboxes`, `sandboxes/status` | create, delete, get, list, patch, update, watch | | `""` | `events` | get, list, watch | | `""` | `pods` | get | +| `apps` | `deployments` | create, delete, get, list, watch | +| `apps` | `replicasets` | get | +| `""` | `services`, `secrets` | create, delete, get, list, watch | +| `networking.k8s.io` | `networkpolicies` | create, delete, get, list, watch | The ClusterRole grants node inspection and token validation: @@ -290,7 +295,7 @@ The gateway exposes `/healthz` for process liveness and `/readyz` for dependency ## Next Steps -- To choose between combined and sidecar sandbox pods, refer to [Topology](/kubernetes/topology). +- To choose between combined, sidecar, and proxy-pod sandbox topology, refer to [Topology](/kubernetes/topology). - To enable automatic certificate rotation with cert-manager, refer to [Managing Certificates](/kubernetes/managing-certificates). - To expose the gateway externally without port-forwarding, refer to [Ingress](/kubernetes/ingress). - To configure OIDC or reverse-proxy authentication, refer to [Access Control](/kubernetes/access-control). diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index 869fc07f1b..456a653d4c 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -3,14 +3,15 @@ # SPDX-License-Identifier: Apache-2.0 title: "Kubernetes Sandbox Topology" sidebar-title: "Topology" -description: "Choose between combined and sidecar supervisor topology for Kubernetes sandbox pods." +description: "Choose between combined, sidecar, and proxy-pod topology for Kubernetes sandbox pods." keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, Sidecar, Network Policy, RuntimeClass" position: 2 --- -Kubernetes sandbox pods can run the OpenShell supervisor in `combined` or -`sidecar` topology. Choose the topology based on which controls you need inside -the pod and how much privilege your cluster allows on the agent container. +Kubernetes sandbox pods can run the OpenShell supervisor in `combined`, +`sidecar`, or `proxy-pod` topology. Choose the topology based on which controls +you need inside the pod, how much privilege your cluster allows on the agent +container, and whether the cluster enforces Kubernetes NetworkPolicies. ## Choose a Topology @@ -22,6 +23,7 @@ lower-privilege agent container. |---|---|---| | `combined` | You need OpenShell network, filesystem, and process controls in the sandbox workload. | The agent container carries the Linux capabilities the supervisor needs. | | `sidecar` | You need the agent container to run as non-root without added Linux capabilities, and network policy is the primary control. | Privilege-dropping and supervisor mount isolation do not run in the agent container. | +| `proxy-pod` | You need network enforcement to run outside the agent pod and your cluster enforces Kubernetes NetworkPolicies. | Requires a NetworkPolicy-enforcing CNI or controller; privilege-dropping and supervisor mount isolation do not run in the agent container. | ## Privilege Model @@ -33,6 +35,8 @@ The long-running container permissions differ by topology: | `sidecar` | Agent container, process-only supervisor (`network-only`) | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Agent and workload run without added Linux capabilities. | | `sidecar` | Network supervisor sidecar, binary-aware mode (default) | `0:sandbox_gid` | `false` | Drops `ALL`; adds `SYS_PTRACE` and `DAC_READ_SEARCH` | Root sidecar inspects cross-UID workload `/proc` entries. The nftables fence exempts UID 0, so do not inject other root containers into these pods. | | `sidecar` | Network supervisor sidecar, endpoint/L7-only mode | `proxyUid:sandbox_gid` | `false` | Drops `ALL` | Non-root sidecar enforces endpoint and L7 policy without matching `policy.binaries`. | +| `proxy-pod` | Agent pod container, process-only supervisor (`network-only`) | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Agent and workload run without added Linux capabilities in their own pod. | +| `proxy-pod` | Supervisor pod container, network proxy only | `proxyPod.proxyUid:sandbox_gid` | `false` | Drops `ALL` | Long-running proxy runs outside the agent pod without added capabilities. | Short-lived setup containers still have the permissions needed to prepare the pod: @@ -41,6 +45,8 @@ pod: |---|---|---|---|---|---| | `combined` | Supervisor install init container | `0` | Not set | Not set | Copies the supervisor binary into the agent container volume. | | `sidecar` | Network init container | `0` | `false` | Drops `ALL`; adds `NET_ADMIN`, `NET_RAW`, `CHOWN`, and `FOWNER` | Installs pod-local nftables rules and prepares shared sidecar state. | +| `proxy-pod` | Supervisor install init container | `0` | Not set | Not set | Copies the supervisor binary into the agent pod volume. | +| `proxy-pod` | Proxy CA install init container | `0:sandbox_gid` | `false` | Drops `ALL` | Copies proxy CA material into the agent pod TLS volume. | ## Combined Topology @@ -158,6 +164,71 @@ Sidecar pods use `shareProcessNamespace: true` so the network sidecar can resolve workload process and binary identity through `/proc/`. +## Proxy-Pod Topology + +Proxy-pod topology moves network enforcement and gateway forwarding into a +separate supervisor Deployment with one pod. The agent pod runs the process +supervisor and reaches the supervisor through a per-sandbox headless Service. + +```mermaid +flowchart TB + Sandbox["agents.x-k8s.io Sandbox"] + + subgraph Namespace["Sandbox namespace"] + subgraph AgentPod["Agent pod"] + ProcessSupervisor["process supervisor
network-only"] + Workload["Agent workload"] + end + + SupervisorDeployment["Supervisor Deployment
1 replica"] + subgraph SupervisorPod["Supervisor pod"] + NetworkProxy["network supervisor proxy
proxyUid"] + end + + Service["Headless Service"] + ProxyCA["Proxy CA Secret"] + AgentEgressPolicy["NetworkPolicy
agent egress to supervisor + DNS"] + SupervisorIngressPolicy["NetworkPolicy
supervisor ingress from paired agent"] + end + + Gateway["OpenShell Gateway"] + External["External services"] + + Sandbox --> AgentPod + Sandbox --> SupervisorDeployment + SupervisorDeployment --> SupervisorPod + ProcessSupervisor --> Workload + AgentPod -->|"egress allowed by NetworkPolicy"| Service + Service --> NetworkProxy + NetworkProxy -->|"gateway forwarding"| Gateway + NetworkProxy -->|"policy-enforced egress"| External + ProxyCA -. mounted .- AgentPod + ProxyCA -. mounted .- SupervisorPod + AgentEgressPolicy -. selects .- AgentPod + SupervisorIngressPolicy -. selects .- SupervisorPod +``` + +OpenShell creates these per-sandbox resources: + +- Agent pod labeled `openshell.ai/sandbox-role=agent`. +- Supervisor Deployment with one pod labeled `openshell.ai/sandbox-role=supervisor`. +- Headless Service for the supervisor pod. +- Proxy CA Secret shared through mounts. +- NetworkPolicy that limits agent egress to the supervisor pod and DNS. +- NetworkPolicy that accepts supervisor ingress only from the paired agent pod. + +The supervisor Deployment has a controlling `Sandbox` ownerReference so +Kubernetes garbage collection removes it when the sandbox is deleted. The +Deployment recreates the supervisor pod if the pod is deleted independently. + + +Proxy-pod topology requires NetworkPolicy enforcement to work as OpenShell +expects. The target cluster must have a policy-enforcing CNI or equivalent +NetworkPolicy controller before deploying this topology. Without enforcement, +the agent pod is not forced through its paired supervisor proxy, so the +agent-to-supervisor isolation policy is only declarative. + + ## Credential Exposure Sidecar topology keeps gateway credentials in the network sidecar. The agent @@ -182,6 +253,11 @@ of the already-running workload entrypoint. Use `combined` topology when you need the full single-supervisor enforcement path; use additional runtime isolation when you need a stronger container boundary around sidecar workloads. +Proxy-pod topology uses a separate supervisor pod for gateway-facing network +enforcement and forwards the agent pod through that supervisor Service. The +proxy-pod agent process supervisor preserves gateway session behavior while +network egress is isolated by the per-sandbox NetworkPolicies described above. + ## RuntimeClass Isolation Sidecar topology has been validated with Kata Containers. It does not currently @@ -195,6 +271,12 @@ mount-isolation controls that sidecar mode relaxes. Use them as an additional workload boundary, not as a replacement for the combined topology's full supervisor controls. +Proxy-pod topology has been tested with Kata Containers and gVisor and is +functional when the cluster enforces NetworkPolicies. Runtime classes do not +re-enable privilege dropping or supervisor mount isolation in `network-only` +process supervision. Use RuntimeClass isolation as an additional workload +boundary, not as a replacement for combined topology. + You can set a default runtime class in the Kubernetes driver configuration or override it per sandbox with driver config: @@ -204,9 +286,10 @@ openshell sandbox create \ -- claude ``` -## Enable Sidecar Mode +## Enable Alternate Topologies -For direct gateway TOML configuration, set the Kubernetes driver fields: +For direct gateway TOML configuration, set the Kubernetes driver fields for +sidecar mode: ```toml [openshell.drivers.kubernetes] @@ -222,7 +305,21 @@ runs the sidecar as UID 0 instead. The network init container exempts the effective sidecar UID from proxy redirection so the sidecar can reach the gateway. -When the Helm chart renders `gateway.toml`, set the equivalent chart values: +Set `topology="proxy-pod"` to use proxy-pod mode: + +```toml +[openshell.drivers.kubernetes] +topology = "proxy-pod" + +[openshell.drivers.kubernetes.proxy_pod] +proxy_uid = 1337 +``` + +`proxy_pod.proxy_uid` must be a non-root UID and must not match the sandbox UID. +It is used by the proxy supervisor pod created by the Deployment. + +When the Helm chart renders `gateway.toml`, set the equivalent chart values for +sidecar mode: ```yaml supervisor: @@ -232,6 +329,15 @@ supervisor: processBinaryAwareNetworkPolicy: true ``` +Set `supervisor.topology=proxy-pod` to use proxy-pod mode: + +```yaml +supervisor: + topology: proxy-pod + proxyPod: + proxyUid: 1337 +``` + Leave `topology` unset, or set it to `combined`, to keep the original single-container supervisor path. For Helm installs, leave `supervisor.topology` unset or set it to `combined`. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index cefae0b5cb..da14287d1d 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -466,6 +466,8 @@ supervisor_sideload_method = "image-volume" # "combined" runs the existing single supervisor container with full process, # filesystem, and network enforcement in the agent container. "sidecar" moves # pod-level network enforcement and gateway session handling into a network sidecar. +# "proxy-pod" moves network enforcement and gateway forwarding into a separate +# supervisor Deployment and uses NetworkPolicy to force agent egress through it. topology = "combined" # Optional corporate HTTP forward proxy for policy-approved TLS egress. The # sandbox workload cannot select or override these settings. Only http:// proxy @@ -546,6 +548,10 @@ proxy_uid = 1337 # inspection capabilities, and enforce endpoint/L7 policy without matching # policy.binaries. process_binary_aware_network_policy = true + +[openshell.drivers.kubernetes.proxy_pod] +# UID used by the network supervisor pod. It must not match the sandbox UID. +proxy_uid = 1337 ``` In managed workspace mode, the Kubernetes driver copies each explicitly named diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 656ae43bb6..6da68ec966 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -379,7 +379,7 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | -| `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | +| `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar, or `proxy-pod` to run network enforcement and gateway forwarding in a separate supervisor Deployment with NetworkPolicy isolation. | | `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | | `no_proxy` | `upstreamProxy.noProxy` | Set destinations that bypass only the corporate proxy. OpenShell policy evaluation still applies. | | `proxy_auth_secret_name` | `upstreamProxy.authSecret.name` | Set the existing Secret name in the sandbox namespace that contains the proxy credential. Requires `sidecar` topology. | @@ -387,7 +387,9 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `proxy_auth_allow_insecure` | `upstreamProxy.authAllowInsecure` | Set `true` to acknowledge that Basic authentication to an HTTP proxy is cleartext. Required with a proxy credential Secret. | | `proxy_connect_by_hostname` | `upstreamProxy.connectByHostname` | Send hostnames rather than validated IPs in CONNECT requests. Use only when proxy ACLs require hostname targets. | | `sidecar.proxy_uid` | `supervisor.sidecar.proxyUid` | Dedicated UID of at least `1000` used by the relaxed sidecar when process/binary-aware network policy is disabled. It must not match the workload UID. The default binary-aware sidecar runs as UID 0. The network init container exempts the effective sidecar UID from proxy redirection. | +| `proxy_pod.proxy_uid` | `supervisor.proxyPod.proxyUid` | Dedicated UID of at least `1000` used by the network supervisor in `proxy-pod` topology. It must not match the workload UID. | | `sidecar.process_binary_aware_network_policy` | `supervisor.sidecar.processBinaryAwareNetworkPolicy` | Keep process/binary-aware network policy enabled in `sidecar` topology. The default runs the sidecar as UID 0 with `SYS_PTRACE` and `DAC_READ_SEARCH`. Set false to run as `proxy_uid`, drop both capabilities, and enforce endpoint/L7 policy without matching `policy.binaries`. | +| `proxy_pod.proxy_uid` | `supervisor.proxyPod.proxyUid` | Non-root UID used by the proxy-pod network supervisor. It must not match the sandbox UID. | | `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | | `workspace_storage_class` | `server.workspaceStorageClass` | Set the `StorageClass` for the workspace PVC. Empty (default) omits `storageClassName` and uses the cluster's default `StorageClass`. Set this on clusters with no default `StorageClass`, otherwise the workspace PVC stays `Pending` and the sandbox never starts. | @@ -425,6 +427,13 @@ identity mount isolation. Network policy still runs in the sidecar, and sidecar pods set `shareProcessNamespace: true` so the network sidecar can resolve process/binary identity through `/proc/`. +In `proxy-pod` topology, network enforcement runs in a separate non-root +supervisor Deployment with one pod, a headless Service, a proxy CA Secret, and +per-sandbox NetworkPolicies. The Deployment recreates the supervisor pod if it +is deleted. The agent process supervisor runs in `network-only` mode; use +`combined` topology when you need combined-mode process/filesystem guards in the +agent container. + The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage. Stop patches the existing resource rather than deleting it. For `v1beta1`, diff --git a/e2e/rust/tests/live_policy_update.rs b/e2e/rust/tests/live_policy_update.rs index 7a1e12923a..555fa01e7d 100644 --- a/e2e/rust/tests/live_policy_update.rs +++ b/e2e/rust/tests/live_policy_update.rs @@ -517,8 +517,10 @@ async fn live_policy_update_from_empty_network_policies() { /// /// NOTE: This exercises the Docker-backed supervisor built from this branch. /// The exact `policy list` status wording ("Loaded"/"Superseded") may differ by -/// CLI version; the assertions below key on the effective version reaching 2 and -/// no revision remaining `Pending` once the acknowledgement lands. +/// CLI version; the assertions below key on the effective version reaching at +/// least 2 and no revision remaining `Pending` once the acknowledgement lands. +/// Multi-supervisor topologies may create a later revision while their network +/// and process leaves reconcile their runtime-specific policy views. #[tokio::test] async fn initial_sparse_policy_is_acknowledged_as_loaded() { // Repo-relative path to the sparse network-only policy fixture. @@ -543,7 +545,8 @@ async fn initial_sparse_policy_is_acknowledged_as_loaded() { // The enriched revision (2) is synced during startup; the acknowledgement // (LOADED) is delivered by the supervisor's poll loop shortly after Ready. - // Poll until the effective policy is version 2 and no revision is Pending. + // Poll until the effective policy is at least version 2 and no revision is + // Pending. A proxy-pod network supervisor may legitimately advance it again. let mut acknowledged = false; let mut last_list = String::new(); for _ in 0..30 { @@ -554,7 +557,7 @@ async fn initial_sparse_policy_is_acknowledged_as_loaded() { last_list = list.output.clone(); let pending = list.output.to_lowercase().contains("pending"); - if version == Some(2) && list.success && !pending { + if version.is_some_and(|version| version >= 2) && list.success && !pending { acknowledged = true; break; } @@ -563,7 +566,7 @@ async fn initial_sparse_policy_is_acknowledged_as_loaded() { assert!( acknowledged, - "enriched initial policy should reach revision 2 with no Pending revision.\n\ + "enriched initial policy should reach at least revision 2 with no Pending revision.\n\ last `policy list` output:\n{last_list}" ); diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index f83c8bafe1..c3a8f5585e 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -20,6 +20,12 @@ # files, relative to the repository root or absolute, to layer additional chart # configuration on top of ci/values-skaffold.yaml. # +# Proxy-pod topology: +# Use OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-proxy-pod.yaml +# or `mise run e2e:kubernetes:proxy-pod`. The target cluster must enforce +# Kubernetes NetworkPolicies; the ephemeral k3d/k3s path keeps k3s's embedded +# network policy controller enabled. +# # Image source: # - Ephemeral k3d mode builds local `openshell/{gateway,supervisor}:${IMAGE_TAG}` # images by default, imports them into k3d, then installs the chart. This @@ -94,6 +100,7 @@ VAULT_CHART_VERSION="${OPENSHELL_E2E_OPENBAO_CHART_VERSION:-0.28.3}" VAULT_DEV_ROOT_TOKEN="${OPENSHELL_E2E_VAULT_DEV_ROOT_TOKEN:-root}" CORPORATE_PROXY_FIXTURE_DEPLOYED=0 CORPORATE_PROXY_FIXTURE_SECRET="openshell-e2e-proxy-auth" +PROXY_POD_E2E=0 # Isolate CLI/SDK gateway metadata from the developer's real config. export XDG_CONFIG_HOME="${WORKDIR}/config" @@ -793,6 +800,9 @@ if [ -n "${OPENSHELL_E2E_KUBE_EXTRA_VALUES:-}" ]; then IFS=':' read -r -a extra_values_files <<< "${OPENSHELL_E2E_KUBE_EXTRA_VALUES}" for values_file in "${extra_values_files[@]}"; do [ -n "${values_file}" ] || continue + if [[ "${values_file}" == *"values-proxy-pod.yaml" ]]; then + PROXY_POD_E2E=1 + fi if [[ "${values_file}" != /* ]]; then values_file="${ROOT}/${values_file}" fi @@ -800,6 +810,11 @@ if [ -n "${OPENSHELL_E2E_KUBE_EXTRA_VALUES:-}" ]; then done fi +if [ "${PROXY_POD_E2E}" = "1" ]; then + echo "Proxy-pod e2e profile enabled; target cluster must enforce Kubernetes NetworkPolicies." + echo "Ephemeral k3d/k3s mode uses k3s's embedded NetworkPolicy controller unless the cluster is customized externally." +fi + if [ "${OPENSHELL_E2E_KUBE_DB_SCENARIOS:-0}" = "1" ]; then # --- Multi-scenario mode: test all database backends --- DB_PASSED=0 diff --git a/tasks/helm.toml b/tasks/helm.toml index 33a61c022c..aa33fae68d 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -68,6 +68,11 @@ description = "Run skaffold dev with the Kubernetes supervisor sidecar topology dir = "deploy/helm/openshell" run = "skaffold dev -p sidecar-mtls" +["helm:skaffold:dev:proxy-pod"] +description = "Run skaffold dev with proxy-pod topology; requires NetworkPolicy enforcement in the target cluster" +dir = "deploy/helm/openshell" +run = "skaffold dev -p proxy-pod" + ["helm:skaffold:run"] description = "Run skaffold run for deploy/helm/openshell (one-shot deploy)" dir = "deploy/helm/openshell" @@ -83,6 +88,11 @@ description = "Run skaffold run with the Kubernetes supervisor sidecar topology dir = "deploy/helm/openshell" run = "skaffold run -p sidecar-mtls" +["helm:skaffold:run:proxy-pod"] +description = "Run skaffold run with proxy-pod topology; requires NetworkPolicy enforcement in the target cluster" +dir = "deploy/helm/openshell" +run = "skaffold run -p proxy-pod" + ["helm:skaffold:delete"] description = "Run skaffold delete for deploy/helm/openshell" dir = "deploy/helm/openshell" @@ -98,6 +108,11 @@ description = "Run skaffold delete for the Kubernetes supervisor sidecar topolog dir = "deploy/helm/openshell" run = "skaffold delete -p sidecar-mtls" +["helm:skaffold:delete:proxy-pod"] +description = "Run skaffold delete for the Kubernetes proxy-pod topology" +dir = "deploy/helm/openshell" +run = "skaffold delete -p proxy-pod" + ["helm:skaffold:diagnose"] description = "Run skaffold diagnose for deploy/helm/openshell" dir = "deploy/helm/openshell" diff --git a/tasks/scripts/helm-k3s-local.sh b/tasks/scripts/helm-k3s-local.sh index 82b8d5cfc8..b2f26b8988 100755 --- a/tasks/scripts/helm-k3s-local.sh +++ b/tasks/scripts/helm-k3s-local.sh @@ -69,6 +69,10 @@ Environment: macOS uses k3d from mise (Docker required). Linux can use this flow only when k3d is installed explicitly; otherwise use kind or an existing cluster context. Pair with: mise run helm:skaffold:dev + +The proxy-pod Skaffold profile relies on Kubernetes NetworkPolicy enforcement. +This helper leaves k3s's embedded network policy controller enabled; if you +replace the CNI, install a policy-enforcing CNI before using that profile. EOF } diff --git a/tasks/test.toml b/tasks/test.toml index a796ea67b4..a431cc28dc 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -160,6 +160,11 @@ description = "Run Kubernetes e2e with the supervisor sidecar topology overlay" env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sidecar.yaml" } run = "e2e/rust/e2e-kubernetes.sh" +["e2e:kubernetes:proxy-pod"] +description = "Run Kubernetes e2e with the proxy-pod topology overlay; requires NetworkPolicy enforcement in the target cluster" +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-proxy-pod.yaml" } +run = "e2e/rust/e2e-kubernetes.sh" + ["e2e:kubernetes:db"] description = "Run Kubernetes e2e with all database backend scenarios (SQLite and external PostgreSQL with existingSecret)" env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" } From c65ea8f5905eec5d18aeca19f5b83093ab6fda80 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Fri, 10 Jul 2026 15:54:59 -0700 Subject: [PATCH 02/25] refactor(kubernetes): run proxy-pod workloads directly Signed-off-by: Taylor Mutch --- crates/openshell-core/src/sandbox_env.rs | 7 +- crates/openshell-driver-kubernetes/README.md | 9 +- .../openshell-driver-kubernetes/src/config.rs | 59 +++- .../openshell-driver-kubernetes/src/driver.rs | 333 ++++++++++-------- crates/openshell-driver-kubernetes/src/lib.rs | 4 +- .../openshell-driver-kubernetes/src/main.rs | 10 +- crates/openshell-sandbox/src/lib.rs | 33 -- deploy/helm/openshell/README.md | 1 + .../openshell/templates/gateway-config.yaml | 1 + .../openshell/tests/gateway_config_test.yaml | 9 + deploy/helm/openshell/values.yaml | 3 + docs/kubernetes/setup.mdx | 1 + docs/kubernetes/topology.mdx | 42 ++- docs/reference/gateway-config.mdx | 2 + docs/reference/sandbox-compute-drivers.mdx | 9 +- 15 files changed, 314 insertions(+), 209 deletions(-) diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 81e6953a43..c512158334 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -26,6 +26,9 @@ pub const SSH_SOCKET_PATH: &str = "OPENSHELL_SSH_SOCKET_PATH"; /// Log level for the sandbox supervisor (e.g. `"debug"`, `"info"`, `"warn"`). pub const LOG_LEVEL: &str = "OPENSHELL_LOG_LEVEL"; +/// Shell command to run inside the sandbox. +pub const SANDBOX_COMMAND: &str = "OPENSHELL_SANDBOX_COMMAND"; + /// Versioned specification for the exact canonical main process. /// /// Most drivers use JSON directly. Transports that cannot preserve spaces in @@ -143,10 +146,6 @@ pub const NETWORK_BINARY_IDENTITY: &str = "OPENSHELL_NETWORK_BINARY_IDENTITY"; /// container. pub const SIDECAR_CONTROL_SOCKET: &str = "OPENSHELL_SIDECAR_CONTROL_SOCKET"; -/// TCP address the process supervisor waits for before starting when the -/// network supervisor runs outside the agent process. -pub const SUPERVISOR_READY_ADDR: &str = "OPENSHELL_SUPERVISOR_READY_ADDR"; - /// Address where an external network supervisor forwards gateway gRPC traffic. pub const GATEWAY_FORWARD_ADDR: &str = "OPENSHELL_GATEWAY_FORWARD_ADDR"; diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 26c4413e81..f92090b8b8 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -137,11 +137,14 @@ restart lifecycle before a new authoritative client can be established. The `proxy-pod` supervisor topology runs network enforcement and gateway forwarding in a separate supervisor Deployment with one pod. The agent pod runs -only the process-mode supervisor and reaches the supervisor through a -per-sandbox headless Service. The driver creates an owner-referenced supervisor +the sandbox image directly and reaches the supervisor through a per-sandbox +headless Service. The driver creates an owner-referenced supervisor Deployment with one replica plus Service, proxy CA Secret, and NetworkPolicy resources so agent egress is limited to its paired supervisor pod plus DNS. If -the supervisor pod is deleted, the Deployment recreates it. +the supervisor pod is deleted, the Deployment recreates it. The workload pod +does not mount gateway credentials or the supervisor binary. This topology +intentionally omits filesystem/process/binary enforcement, SSH/exec, +upload/download, sync, and provider environment injection. The driver can request a Kubernetes AppArmor profile through `app_armor_profile`. diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 5284ee0131..2e96a8901e 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -182,18 +182,61 @@ impl KubernetesSidecarConfig { } } +/// Scheduling relationship between a proxy-pod workload and its paired +/// network-supervisor pod. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ProxyPodAffinity { + /// Do not add an OpenShell-managed pod-affinity term. + #[default] + Disabled, + /// Prefer same-node placement without making it a scheduling requirement. + Preferred, + /// Require the workload and network supervisor to run on the same node. + Required, +} + +impl std::fmt::Display for ProxyPodAffinity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Disabled => f.write_str("disabled"), + Self::Preferred => f.write_str("preferred"), + Self::Required => f.write_str("required"), + } + } +} + +impl FromStr for ProxyPodAffinity { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "disabled" => Ok(Self::Disabled), + "preferred" => Ok(Self::Preferred), + "required" => Ok(Self::Required), + other => Err(format!( + "unknown proxy-pod affinity '{other}'; expected 'disabled', 'preferred', or 'required'" + )), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesProxyPodConfig { /// UID used by the network supervisor in `proxy-pod` topology. It must not /// match the sandbox workload UID. pub proxy_uid: u32, + /// Whether same-node placement with the paired supervisor is disabled, + /// preferred, or required. + pub affinity: ProxyPodAffinity, } impl Default for KubernetesProxyPodConfig { fn default() -> Self { Self { proxy_uid: DEFAULT_PROXY_UID, + affinity: ProxyPodAffinity::Disabled, } } } @@ -957,6 +1000,7 @@ mod tests { fn default_proxy_uid_is_dedicated_non_root_uid() { let cfg = KubernetesComputeConfig::default(); assert_eq!(cfg.sidecar.proxy_uid, DEFAULT_PROXY_UID); + assert_eq!(cfg.proxy_pod.affinity, ProxyPodAffinity::Disabled); } #[test] @@ -997,14 +1041,27 @@ mod tests { fn serde_override_proxy_pod_proxy_uid_nested() { let json = serde_json::json!({ "proxy_pod": { - "proxy_uid": 2000 + "proxy_uid": 2000, + "affinity": "preferred" } }); let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); assert_eq!(cfg.proxy_pod.proxy_uid, 2000); + assert_eq!(cfg.proxy_pod.affinity, ProxyPodAffinity::Preferred); cfg.validate_proxy_uid().unwrap(); } + #[test] + fn serde_rejects_invalid_proxy_pod_affinity() { + let json = serde_json::json!({ + "proxy_pod": { + "affinity": "sometimes" + } + }); + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown variant")); + } + #[test] fn serde_rejects_sidecar_binary_identity_field() { let json = serde_json::json!({ diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 1d607e1902..66a158cb99 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -7,8 +7,8 @@ use super::AppArmorProfile; use crate::config::{ DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, OperatorNamespaceAllowlist, - SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, is_dns_1123_label, - managed_namespace, validate_managed_namespace_name, + ProxyPodAffinity, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, + is_dns_1123_label, managed_namespace, validate_managed_namespace_name, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::apps::v1::Deployment; @@ -1425,6 +1425,7 @@ impl KubernetesComputeDriver { proxy_auth_secret_key: self.config.proxy_auth_secret_key.as_deref(), proxy_auth_allow_insecure: self.config.proxy_auth_allow_insecure == Some(true), proxy_connect_by_hostname: self.config.proxy_connect_by_hostname == Some(true), + proxy_pod_affinity: self.config.proxy_pod.affinity, namespace: &self.config.namespace, service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, @@ -2576,7 +2577,6 @@ const PROXY_POD_NETWORK_ENFORCEMENT_MODE: &str = "proxy-pod"; const PROXY_POD_CA_SECRET_MOUNT_PATH: &str = "/var/run/openshell-proxy-ca"; const PROXY_POD_CA_CERT_FILE: &str = "openshell-ca.pem"; const PROXY_POD_CA_KEY_FILE: &str = "openshell-ca-key.pem"; -const PROXY_POD_SSH_SOCKET_FILE: &str = "/tmp/openshell/ssh.sock"; /// Build the emptyDir volume that holds the supervisor binary. /// @@ -2868,22 +2868,6 @@ fn sidecar_tls_volume_mount() -> serde_json::Value { }) } -fn gateway_tls_server_name(grpc_endpoint: &str) -> Option { - let rest = grpc_endpoint.strip_prefix("https://")?; - let authority = rest.split('/').next().unwrap_or(rest); - if authority.is_empty() { - return None; - } - if let Some(bracketed) = authority.strip_prefix('[') { - return bracketed.split(']').next().map(str::to_string); - } - authority - .split(':') - .next() - .filter(|host| !host.is_empty()) - .map(str::to_string) -} - #[derive(Debug, Clone)] struct ProxyPodResourceNames { supervisor_deployment: String, @@ -2943,16 +2927,6 @@ fn proxy_pod_service_dns(service_name: &str, namespace: &str) -> String { format!("{service_name}.{namespace}.svc.cluster.local") } -fn proxy_pod_process_gateway_endpoint(service_dns: &str, grpc_endpoint: &str) -> String { - if grpc_endpoint.is_empty() { - String::new() - } else if grpc_endpoint.starts_with("https://") { - format!("https://{service_dns}:{PROXY_POD_GATEWAY_FORWARD_PORT}") - } else { - format!("http://{service_dns}:{PROXY_POD_GATEWAY_FORWARD_PORT}") - } -} - fn proxy_pod_proxy_url(service_dns: &str) -> String { format!("http://{service_dns}:{PROXY_POD_PROXY_PORT}") } @@ -3408,11 +3382,19 @@ fn proxy_pod_ca_init_container( fn apply_proxy_pod_affinity( spec: &mut serde_json::Map, sandbox_id: &str, + mode: ProxyPodAffinity, ) { - if sandbox_id.is_empty() { + if sandbox_id.is_empty() || mode == ProxyPodAffinity::Disabled { return; } + let term = serde_json::json!({ + "labelSelector": { + "matchLabels": proxy_pod_match_labels(sandbox_id, SANDBOX_ROLE_SUPERVISOR) + }, + "topologyKey": "kubernetes.io/hostname" + }); + let affinity = spec .entry("affinity".to_string()) .or_insert_with(|| serde_json::json!({})); @@ -3431,19 +3413,33 @@ fn apply_proxy_pod_affinity( let pod_affinity = pod_affinity .as_object_mut() .expect("podAffinity was converted to object"); - let required = pod_affinity - .entry("requiredDuringSchedulingIgnoredDuringExecution".to_string()) - .or_insert_with(|| serde_json::json!([])); - if !required.is_array() { - *required = serde_json::json!([]); - } - if let Some(required) = required.as_array_mut() { - required.push(serde_json::json!({ - "labelSelector": { - "matchLabels": proxy_pod_match_labels(sandbox_id, SANDBOX_ROLE_SUPERVISOR) - }, - "topologyKey": "kubernetes.io/hostname" - })); + match mode { + ProxyPodAffinity::Disabled => {} + ProxyPodAffinity::Preferred => { + let preferred = pod_affinity + .entry("preferredDuringSchedulingIgnoredDuringExecution".to_string()) + .or_insert_with(|| serde_json::json!([])); + if !preferred.is_array() { + *preferred = serde_json::json!([]); + } + if let Some(preferred) = preferred.as_array_mut() { + preferred.push(serde_json::json!({ + "weight": 100, + "podAffinityTerm": term, + })); + } + } + ProxyPodAffinity::Required => { + let required = pod_affinity + .entry("requiredDuringSchedulingIgnoredDuringExecution".to_string()) + .or_insert_with(|| serde_json::json!([])); + if !required.is_array() { + *required = serde_json::json!([]); + } + if let Some(required) = required.as_array_mut() { + required.push(term); + } + } } } @@ -3462,14 +3458,7 @@ fn apply_supervisor_proxy_pod_topology( sc.insert("fsGroup".to_string(), serde_json::json!(params.sandbox_gid)); } - apply_supervisor_binary_source( - spec, - params.supervisor_image, - params.supervisor_image_pull_policy, - params.supervisor_sideload_method, - ); - - apply_proxy_pod_affinity(spec, params.sandbox_id); + apply_proxy_pod_affinity(spec, params.sandbox_id, params.proxy_pod_affinity); let names = proxy_pod_resource_names(params.sandbox_name); let service_dns = proxy_pod_service_dns(&names.service, params.namespace); @@ -3496,22 +3485,14 @@ fn apply_supervisor_proxy_pod_topology( })); } - let image = spec - .get("containers") - .and_then(|v| v.as_array()) - .and_then(|containers| containers.first()) - .and_then(|container| container.get("image")) - .and_then(|value| value.as_str()) - .unwrap_or(params.default_image) - .to_string(); let init_containers = spec .entry("initContainers") .or_insert_with(|| serde_json::json!([])) .as_array_mut(); if let Some(init_containers) = init_containers { init_containers.push(proxy_pod_ca_init_container( - &image, - params.image_pull_policy, + params.supervisor_image, + params.supervisor_image_pull_policy, params.sandbox_gid, )); } @@ -3527,17 +3508,12 @@ fn apply_supervisor_proxy_pod_topology( .get_mut(target_index) .and_then(|v| v.as_object_mut()) { - container.insert( - "command".to_string(), - serde_json::json!([ - format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), - "--mode=process" - ]), - ); - let security_context = container .entry("securityContext") .or_insert_with(|| serde_json::json!({})); + if !security_context.is_object() { + *security_context = serde_json::json!({}); + } if let Some(sc) = security_context.as_object_mut() { sc.insert( "runAsUser".to_string(), @@ -3554,9 +3530,7 @@ fn apply_supervisor_proxy_pod_topology( ); sc.insert( "capabilities".to_string(), - serde_json::json!({ - "drop": ["ALL"] - }), + serde_json::json!({ "drop": ["ALL"] }), ); } @@ -3565,7 +3539,9 @@ fn apply_supervisor_proxy_pod_topology( .or_insert_with(|| serde_json::json!([])) .as_array_mut(); if let Some(volume_mounts) = volume_mounts { - volume_mounts.push(supervisor_volume_mount()); + remove_volume_mount(volume_mounts, SERVICE_ACCOUNT_TOKEN_VOLUME_NAME); + remove_volume_mount(volume_mounts, CLIENT_TLS_VOLUME_NAME); + remove_volume_mount(volume_mounts, SPIFFE_WORKLOAD_API_VOLUME_NAME); volume_mounts.push(proxy_pod_ca_tls_volume_mount()); } @@ -3574,62 +3550,68 @@ fn apply_supervisor_proxy_pod_topology( .or_insert_with(|| serde_json::json!([])) .as_array_mut(); if let Some(env) = env { - let process_endpoint = - proxy_pod_process_gateway_endpoint(&service_dns, params.grpc_endpoint); - upsert_env( - env, + for name in [ + openshell_core::sandbox_env::SANDBOX_ID, + openshell_core::sandbox_env::SANDBOX, openshell_core::sandbox_env::ENDPOINT, - &process_endpoint, - ); - if let Some(server_name) = gateway_tls_server_name(params.grpc_endpoint) { - upsert_env( - env, - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, - &server_name, - ); - } - upsert_env( - env, - openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, - "proxy-pod", - ); - upsert_env( - env, - openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, - PROXY_POD_NETWORK_ENFORCEMENT_MODE, - ); - upsert_env( - env, + openshell_core::sandbox_env::SANDBOX_COMMAND, + openshell_core::sandbox_env::TELEMETRY_ENABLED, openshell_core::sandbox_env::SSH_SOCKET_PATH, - PROXY_POD_SSH_SOCKET_FILE, - ); - upsert_env( - env, - openshell_core::sandbox_env::PROXY_URL, - &proxy_pod_proxy_url(&service_dns), - ); - upsert_env( - env, - openshell_core::sandbox_env::SUPERVISOR_READY_ADDR, - &format!("{service_dns}:{PROXY_POD_PROXY_PORT}"), - ); - upsert_env( - env, - openshell_core::sandbox_env::PROXY_TLS_DIR, - SIDECAR_TLS_MOUNT_PATH, - ); - upsert_env( - env, - openshell_core::sandbox_env::SANDBOX_UID, - ¶ms.sandbox_uid.to_string(), - ); - upsert_env( - env, - openshell_core::sandbox_env::SANDBOX_GID, - ¶ms.sandbox_gid.to_string(), - ); + openshell_core::sandbox_env::TLS_CA, + openshell_core::sandbox_env::TLS_CERT, + openshell_core::sandbox_env::TLS_KEY, + openshell_core::sandbox_env::K8S_SA_TOKEN_FILE, + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + ] { + remove_env(env, name); + } + let proxy_url = proxy_pod_proxy_url(&service_dns); + for name in [ + "ALL_PROXY", + "HTTP_PROXY", + "HTTPS_PROXY", + "http_proxy", + "https_proxy", + "grpc_proxy", + ] { + upsert_env(env, name, &proxy_url); + } + for name in ["NO_PROXY", "no_proxy"] { + upsert_env(env, name, "127.0.0.1,localhost,::1"); + } + upsert_env(env, "NODE_USE_ENV_PROXY", "1"); + + let ca_cert = format!("{SIDECAR_TLS_MOUNT_PATH}/{PROXY_POD_CA_CERT_FILE}"); + let ca_bundle = format!("{SIDECAR_TLS_MOUNT_PATH}/ca-bundle.pem"); + for name in ["NODE_EXTRA_CA_CERTS", "DENO_CERT"] { + upsert_env(env, name, &ca_cert); + } + for name in [ + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CURL_CA_BUNDLE", + "GIT_SSL_CAINFO", + ] { + upsert_env(env, name, &ca_bundle); + } } } + + if let Some(volumes) = spec + .get_mut("volumes") + .and_then(|value| value.as_array_mut()) + { + volumes.retain(|volume| { + !matches!( + volume.get("name").and_then(|value| value.as_str()), + Some( + SERVICE_ACCOUNT_TOKEN_VOLUME_NAME + | CLIENT_TLS_VOLUME_NAME + | SPIFFE_WORKLOAD_API_VOLUME_NAME + ) + ) + }); + } } /// Apply workspace persistence transforms to an already-built pod template. @@ -3803,6 +3785,7 @@ struct SandboxPodParams<'a> { proxy_auth_secret_key: Option<&'a str>, proxy_auth_allow_insecure: bool, proxy_connect_by_hostname: bool, + proxy_pod_affinity: ProxyPodAffinity, namespace: &'a str, service_account_name: &'a str, sandbox_id: &'a str, @@ -3845,6 +3828,7 @@ impl Default for SandboxPodParams<'_> { proxy_auth_secret_key: None, proxy_auth_allow_insecure: false, proxy_connect_by_hostname: false, + proxy_pod_affinity: ProxyPodAffinity::Disabled, namespace: "default", service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, sandbox_id: "", @@ -7214,7 +7198,7 @@ mod tests { } #[test] - fn proxy_pod_topology_renders_process_agent_with_proxy_service() { + fn proxy_pod_topology_runs_workload_directly_through_proxy_service() { let params = SandboxPodParams { topology: SupervisorTopology::ProxyPod, supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, @@ -7248,36 +7232,30 @@ mod tests { pod_template["metadata"]["labels"][LABEL_SANDBOX_ROLE], SANDBOX_ROLE_AGENT ); - assert_eq!( - agent["command"], - serde_json::json!([ - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), - "--mode=process" - ]) - ); + assert!(agent.get("command").is_none()); assert_eq!( rendered_env(agent, openshell_core::sandbox_env::ENDPOINT), - Some(format!("https://{service_dns}:18080").as_str()) - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME), - Some("openshell-gateway.openshell.svc") + None ); assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::PROXY_URL), + rendered_env(agent, "HTTP_PROXY"), Some(format!("http://{service_dns}:3128").as_str()) ); assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SUPERVISOR_READY_ADDR), - Some(format!("{service_dns}:3128").as_str()) + rendered_env(agent, "SSL_CERT_FILE"), + Some("/etc/openshell-tls/proxy/ca-bundle.pem") ); assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE), - Some(PROXY_POD_NETWORK_ENFORCEMENT_MODE) + rendered_env(agent, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE), + None ); assert_eq!( rendered_env(agent, openshell_core::sandbox_env::SSH_SOCKET_PATH), - Some(PROXY_POD_SSH_SOCKET_FILE) + None + ); + assert_eq!( + agent["securityContext"]["capabilities"]["drop"], + serde_json::json!(["ALL"]) ); let containers = pod_template["spec"]["containers"].as_array().unwrap(); @@ -7290,14 +7268,73 @@ mod tests { assert!(volumes.iter().any(|volume| { volume["name"] == "openshell-proxy-pod-tls" && volume["emptyDir"].is_object() })); + assert!(!volumes.iter().any(|volume| { + matches!( + volume["name"].as_str(), + Some( + SUPERVISOR_VOLUME_NAME + | SERVICE_ACCOUNT_TOKEN_VOLUME_NAME + | CLIENT_TLS_VOLUME_NAME + | SPIFFE_WORKLOAD_API_VOLUME_NAME + ) + ) + })); + + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + assert!(init_containers.iter().any(|container| { + container["name"] == "openshell-proxy-ca-install" + && container["image"] == "supervisor-image:latest" + })); + assert!( + !init_containers + .iter() + .any(|container| container["name"] == SUPERVISOR_INIT_CONTAINER_NAME) + ); - let affinity = &pod_template["spec"]["affinity"]["podAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] - [0]; + assert!(pod_template["spec"].get("affinity").is_none()); + } + + #[test] + fn proxy_pod_topology_supports_preferred_affinity() { + let mut spec = serde_json::Map::new(); + apply_proxy_pod_affinity(&mut spec, "sandbox-123", ProxyPodAffinity::Preferred); + + let preferred = + &spec["affinity"]["podAffinity"]["preferredDuringSchedulingIgnoredDuringExecution"][0]; + assert_eq!(preferred["weight"], 100); assert_eq!( - affinity["labelSelector"]["matchLabels"][LABEL_SANDBOX_ROLE], + preferred["podAffinityTerm"]["labelSelector"]["matchLabels"][LABEL_SANDBOX_ROLE], SANDBOX_ROLE_SUPERVISOR ); - assert_eq!(affinity["topologyKey"], "kubernetes.io/hostname"); + assert_eq!( + preferred["podAffinityTerm"]["topologyKey"], + "kubernetes.io/hostname" + ); + } + + #[test] + fn proxy_pod_topology_supports_required_affinity_without_replacing_existing_terms() { + let mut spec = serde_json::json!({ + "affinity": { + "podAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": [{ + "topologyKey": "topology.kubernetes.io/zone" + }] + } + } + }) + .as_object() + .unwrap() + .clone(); + apply_proxy_pod_affinity(&mut spec, "sandbox-123", ProxyPodAffinity::Required); + + let required = + spec["affinity"]["podAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] + .as_array() + .unwrap(); + assert_eq!(required.len(), 2); + assert_eq!(required[0]["topologyKey"], "topology.kubernetes.io/zone"); + assert_eq!(required[1]["topologyKey"], "kubernetes.io/hostname"); } #[test] diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index f994a7663d..99a8aa2487 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -8,8 +8,8 @@ pub mod grpc; pub use config::{ AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesProxyPodConfig, - KubernetesSidecarConfig, ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, - WorkspaceMode, managed_namespace_prefix, + KubernetesSidecarConfig, ManagedSshIngressConfig, ProxyPodAffinity, SupervisorSideloadMethod, + SupervisorTopology, WorkspaceMode, managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index cc7990558d..fdd8e2cdd4 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -14,7 +14,7 @@ use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServ use openshell_driver_kubernetes::{ AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, - KubernetesProxyPodConfig, KubernetesSidecarConfig, ManagedSshIngressConfig, + KubernetesProxyPodConfig, KubernetesSidecarConfig, ManagedSshIngressConfig, ProxyPodAffinity, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, }; @@ -169,6 +169,13 @@ struct Args { )] proxy_pod_proxy_uid: u32, + #[arg( + long = "proxy-pod-affinity", + env = "OPENSHELL_K8S_PROXY_POD_AFFINITY", + default_value = "disabled" + )] + proxy_pod_affinity: ProxyPodAffinity, + #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, @@ -267,6 +274,7 @@ async fn main() -> Result<()> { }, proxy_pod: KubernetesProxyPodConfig { proxy_uid: args.proxy_pod_proxy_uid, + affinity: args.proxy_pod_affinity, }, https_proxy: args.https_proxy, no_proxy: args.no_proxy, diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 9dd10a736b..c8cb395a17 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -169,15 +169,6 @@ pub async fn run_sandbox( } else { None }; - let supervisor_ready_addr = supervisor_ready_addr(); - if process_enabled - && !network_enabled - && proxy_pod_network_enforcement - && let Some(addr) = supervisor_ready_addr.as_deref() - { - wait_for_supervisor_ready_addr(addr).await?; - } - // Extension credentials are owned by this supervisor and shared by every // gateway connection it opens, so the middleware registry's bearer slots // and the policy poll loop that rotates them stay the same objects. @@ -1070,30 +1061,6 @@ fn sidecar_control_socket() -> Option { .map(std::path::PathBuf::from) } -fn supervisor_ready_addr() -> Option { - std::env::var(openshell_core::sandbox_env::SUPERVISOR_READY_ADDR) - .ok() - .filter(|value| !value.is_empty()) -} - -async fn wait_for_supervisor_ready_addr(addr: &str) -> Result<()> { - let deadline = tokio::time::Instant::now() + Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS); - loop { - match TcpStream::connect(addr).await { - Ok(_) => { - info!(addr, "Network supervisor TCP endpoint is ready"); - return Ok(()); - } - Err(err) if tokio::time::Instant::now() >= deadline => { - return Err(miette::miette!( - "timed out waiting for network supervisor TCP endpoint {addr}: {err}" - )); - } - Err(_) => tokio::time::sleep(Duration::from_millis(250)).await, - } - } -} - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] fn sidecar_expected_peer() -> Result { fn required_numeric_env(name: &str) -> Result { diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index d120122bde..a64488b289 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -284,6 +284,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | +| supervisor.proxyPod.affinity | string | `"disabled"` | Same-node scheduling relationship between the workload pod and its paired proxy supervisor: disabled, preferred, or required. | | supervisor.proxyPod.proxyUid | int | `1337` | UID for the network supervisor in proxy-pod topology. The configured UID must not match the sandbox UID. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | | supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 23fbfa5229..6d182ced46 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -208,6 +208,7 @@ data: [openshell.drivers.kubernetes.proxy_pod] proxy_uid = {{ .Values.supervisor.proxyPod.proxyUid | default 1337 }} + affinity = {{ .Values.supervisor.proxyPod.affinity | default "disabled" | quote }} {{- if not $credentialDrivers }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index fd100718f4..afa3dd4636 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -220,6 +220,15 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\.proxy_pod\].*?proxy_uid\s*=\s*2300' + - it: renders proxy pod affinity under [openshell.drivers.kubernetes.proxy_pod] + template: templates/gateway-config.yaml + set: + supervisor.proxyPod.affinity: preferred + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.proxy_pod\].*?affinity\s*=\s*"preferred"' + - it: renders process binary aware network policy under [openshell.drivers.kubernetes.sidecar] template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 324a6dad78..95bfc14dbd 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -67,6 +67,9 @@ supervisor: # -- UID for the network supervisor in proxy-pod topology. The configured # UID must not match the sandbox UID. proxyUid: 1337 + # -- Same-node scheduling relationship between the workload pod and its + # paired proxy supervisor: disabled, preferred, or required. + affinity: disabled # -- Operator-owned corporate forward proxy for policy-approved TLS egress # from Kubernetes sandboxes. The workload cannot select or override it. diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index 21cd0828e9..216662dcd9 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -178,6 +178,7 @@ The most commonly changed values are: | `supervisor.sidecar.proxyUid` | Non-root UID used when sidecar process/binary-aware network policy is disabled. The default binary-aware sidecar runs as UID 0 instead. The configured UID must not match the sandbox UID. | | `upstreamProxy` | Operator-owned corporate HTTP forward proxy for policy-approved TLS egress. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). | | `supervisor.proxyPod.proxyUid` | Non-root UID used by the proxy-pod network supervisor. The UID must not match the sandbox UID. | +| `supervisor.proxyPod.affinity` | Same-node placement policy for workload and proxy pods: `disabled` (default), `preferred`, or `required`. | Use a values file for repeatable deployments: diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index 456a653d4c..dfcbf8f22a 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -23,7 +23,7 @@ lower-privilege agent container. |---|---|---| | `combined` | You need OpenShell network, filesystem, and process controls in the sandbox workload. | The agent container carries the Linux capabilities the supervisor needs. | | `sidecar` | You need the agent container to run as non-root without added Linux capabilities, and network policy is the primary control. | Privilege-dropping and supervisor mount isolation do not run in the agent container. | -| `proxy-pod` | You need network enforcement to run outside the agent pod and your cluster enforces Kubernetes NetworkPolicies. | Requires a NetworkPolicy-enforcing CNI or controller; privilege-dropping and supervisor mount isolation do not run in the agent container. | +| `proxy-pod` | You need network enforcement outside the agent pod, accept a workload-only sandbox container, and your cluster enforces Kubernetes NetworkPolicies. | No sandbox supervisor: filesystem/process/binary controls, SSH, exec, upload/download, sync, and provider injection are unavailable. | ## Privilege Model @@ -35,7 +35,7 @@ The long-running container permissions differ by topology: | `sidecar` | Agent container, process-only supervisor (`network-only`) | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Agent and workload run without added Linux capabilities. | | `sidecar` | Network supervisor sidecar, binary-aware mode (default) | `0:sandbox_gid` | `false` | Drops `ALL`; adds `SYS_PTRACE` and `DAC_READ_SEARCH` | Root sidecar inspects cross-UID workload `/proc` entries. The nftables fence exempts UID 0, so do not inject other root containers into these pods. | | `sidecar` | Network supervisor sidecar, endpoint/L7-only mode | `proxyUid:sandbox_gid` | `false` | Drops `ALL` | Non-root sidecar enforces endpoint and L7 policy without matching `policy.binaries`. | -| `proxy-pod` | Agent pod container, process-only supervisor (`network-only`) | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Agent and workload run without added Linux capabilities in their own pod. | +| `proxy-pod` | Agent pod workload container | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Runs the sandbox image directly with proxy and CA environment only. | | `proxy-pod` | Supervisor pod container, network proxy only | `proxyPod.proxyUid:sandbox_gid` | `false` | Drops `ALL` | Long-running proxy runs outside the agent pod without added capabilities. | Short-lived setup containers still have the permissions needed to prepare the @@ -45,8 +45,7 @@ pod: |---|---|---|---|---|---| | `combined` | Supervisor install init container | `0` | Not set | Not set | Copies the supervisor binary into the agent container volume. | | `sidecar` | Network init container | `0` | `false` | Drops `ALL`; adds `NET_ADMIN`, `NET_RAW`, `CHOWN`, and `FOWNER` | Installs pod-local nftables rules and prepares shared sidecar state. | -| `proxy-pod` | Supervisor install init container | `0` | Not set | Not set | Copies the supervisor binary into the agent pod volume. | -| `proxy-pod` | Proxy CA install init container | `0:sandbox_gid` | `false` | Drops `ALL` | Copies proxy CA material into the agent pod TLS volume. | +| `proxy-pod` | Proxy CA install init container | `0:sandbox_gid` | `false` | Drops `ALL` | Uses the supervisor utility image to copy proxy CA material into the agent pod TLS volume. | ## Combined Topology @@ -167,8 +166,9 @@ resolve workload process and binary identity through `/proc/`. ## Proxy-Pod Topology Proxy-pod topology moves network enforcement and gateway forwarding into a -separate supervisor Deployment with one pod. The agent pod runs the process -supervisor and reaches the supervisor through a per-sandbox headless Service. +separate supervisor Deployment with one pod. The agent pod runs the workload +image directly and reaches the supervisor through a per-sandbox +headless Service. ```mermaid flowchart TB @@ -176,8 +176,7 @@ flowchart TB subgraph Namespace["Sandbox namespace"] subgraph AgentPod["Agent pod"] - ProcessSupervisor["process supervisor
network-only"] - Workload["Agent workload"] + Workload["Sandbox workload
runs image directly"] end SupervisorDeployment["Supervisor Deployment
1 replica"] @@ -197,7 +196,6 @@ flowchart TB Sandbox --> AgentPod Sandbox --> SupervisorDeployment SupervisorDeployment --> SupervisorPod - ProcessSupervisor --> Workload AgentPod -->|"egress allowed by NetworkPolicy"| Service Service --> NetworkProxy NetworkProxy -->|"gateway forwarding"| Gateway @@ -221,6 +219,19 @@ The supervisor Deployment has a controlling `Sandbox` ownerReference so Kubernetes garbage collection removes it when the sandbox is deleted. The Deployment recreates the supervisor pod if the pod is deleted independently. +Same-node scheduling is disabled by default. Set `proxy_pod.affinity` (or Helm +`supervisor.proxyPod.affinity`) to `preferred` for soft same-node placement or +`required` for hard same-node placement. Both modes match the paired supervisor +on `kubernetes.io/hostname` and preserve any affinity supplied by the workload. + +The agent pod does not mount or execute the OpenShell supervisor. The driver +injects standard proxy variables and proxy CA trust directly into the workload +container. Consequently, proxy-pod topology provides network enforcement only: +OpenShell filesystem policy, process controls, binary identity, SSH/connect, +exec, upload/download, file sync, dynamic provider environment injection, and +other process-supervisor features are unavailable. The sandbox image's own +entrypoint and command determine what runs. + Proxy-pod topology requires NetworkPolicy enforcement to work as OpenShell expects. The target cluster must have a policy-enforcing CNI or equivalent @@ -255,8 +266,9 @@ isolation when you need a stronger container boundary around sidecar workloads. Proxy-pod topology uses a separate supervisor pod for gateway-facing network enforcement and forwards the agent pod through that supervisor Service. The -proxy-pod agent process supervisor preserves gateway session behavior while -network egress is isolated by the per-sandbox NetworkPolicies described above. +workload pod receives no gateway endpoint, bootstrap token, client TLS identity, +or SPIFFE workload socket. Network egress is isolated by the per-sandbox +NetworkPolicies described above. ## RuntimeClass Isolation @@ -273,9 +285,9 @@ supervisor controls. Proxy-pod topology has been tested with Kata Containers and gVisor and is functional when the cluster enforces NetworkPolicies. Runtime classes do not -re-enable privilege dropping or supervisor mount isolation in `network-only` -process supervision. Use RuntimeClass isolation as an additional workload -boundary, not as a replacement for combined topology. +restore the supervisor features omitted from the workload pod. Use RuntimeClass +isolation as an additional workload boundary, not as a replacement for combined +topology. You can set a default runtime class in the Kubernetes driver configuration or override it per sandbox with driver config: @@ -313,6 +325,7 @@ topology = "proxy-pod" [openshell.drivers.kubernetes.proxy_pod] proxy_uid = 1337 +affinity = "disabled" # disabled | preferred | required ``` `proxy_pod.proxy_uid` must be a non-root UID and must not match the sandbox UID. @@ -336,6 +349,7 @@ supervisor: topology: proxy-pod proxyPod: proxyUid: 1337 + affinity: disabled ``` Leave `topology` unset, or set it to `combined`, to keep the original diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index da14287d1d..c54dddc00f 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -552,6 +552,8 @@ process_binary_aware_network_policy = true [openshell.drivers.kubernetes.proxy_pod] # UID used by the network supervisor pod. It must not match the sandbox UID. proxy_uid = 1337 +# Same-node workload/supervisor placement: disabled, preferred, or required. +affinity = "disabled" ``` In managed workspace mode, the Kubernetes driver copies each explicitly named diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 6da68ec966..6bda8740c1 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -390,6 +390,7 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `proxy_pod.proxy_uid` | `supervisor.proxyPod.proxyUid` | Dedicated UID of at least `1000` used by the network supervisor in `proxy-pod` topology. It must not match the workload UID. | | `sidecar.process_binary_aware_network_policy` | `supervisor.sidecar.processBinaryAwareNetworkPolicy` | Keep process/binary-aware network policy enabled in `sidecar` topology. The default runs the sidecar as UID 0 with `SYS_PTRACE` and `DAC_READ_SEARCH`. Set false to run as `proxy_uid`, drop both capabilities, and enforce endpoint/L7 policy without matching `policy.binaries`. | | `proxy_pod.proxy_uid` | `supervisor.proxyPod.proxyUid` | Non-root UID used by the proxy-pod network supervisor. It must not match the sandbox UID. | +| `proxy_pod.affinity` | `supervisor.proxyPod.affinity` | Configure same-node workload/supervisor placement as `disabled` (default), `preferred`, or `required`. | | `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | | `workspace_storage_class` | `server.workspaceStorageClass` | Set the `StorageClass` for the workspace PVC. Empty (default) omits `storageClassName` and uses the cluster's default `StorageClass`. Set this on clusters with no default `StorageClass`, otherwise the workspace PVC stays `Pending` and the sandbox never starts. | @@ -430,9 +431,11 @@ process/binary identity through `/proc/`. In `proxy-pod` topology, network enforcement runs in a separate non-root supervisor Deployment with one pod, a headless Service, a proxy CA Secret, and per-sandbox NetworkPolicies. The Deployment recreates the supervisor pod if it -is deleted. The agent process supervisor runs in `network-only` mode; use -`combined` topology when you need combined-mode process/filesystem guards in the -agent container. +is deleted. The sandbox container runs its image directly with proxy and CA +environment; it does not mount or execute the supervisor. Filesystem/process +policy, binary detection, SSH/exec, upload/download, sync, and provider +environment injection are therefore unavailable. Use `combined` or `sidecar` +when those process-supervisor features are required. The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage. From 77ac3b7560c68eacd98793f5fedc5bd46ed1a714 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 13 Jul 2026 13:13:04 -0700 Subject: [PATCH 03/25] fix(kubernetes): harden proxy-pod workloads Signed-off-by: Taylor Mutch --- architecture/gateway.md | 5 +- crates/openshell-driver-kubernetes/README.md | 5 +- .../openshell-driver-kubernetes/src/driver.rs | 157 ++++++++++++++++-- deploy/helm/openshell/templates/role.yaml | 8 - .../tests/sandbox_namespace_test.yaml | 10 -- docs/kubernetes/topology.mdx | 8 +- 6 files changed, 156 insertions(+), 37 deletions(-) diff --git a/architecture/gateway.md b/architecture/gateway.md index 829c4e13c8..59f55cb71a 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -226,7 +226,10 @@ minting the gateway JWT. Agent pods must be directly controlled by the `Pod -> ReplicaSet -> Deployment -> Sandbox` chain. The bootstrap path accepts both `agents.x-k8s.io/v1beta1` ownerReferences from newer Agent Sandbox controllers and `agents.x-k8s.io/v1alpha1` ownerReferences from existing -deployments. Supervisors renew gateway JWTs in memory before expiry only while +deployments. The proxy-pod gateway Role grants create/delete on its dependent +Service, Secret, and NetworkPolicy resources, plus create/delete/get on the +supervisor Deployment and get on its ReplicaSet for this owner-chain check. +Supervisors renew gateway JWTs in memory before expiry only while the sandbox record still exists. Older tokens are not server-revoked; shared deployments bound replay exposure with short `gateway_jwt.ttl_secs` lifetimes. The config default is diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index f92090b8b8..0f04d3f960 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -142,7 +142,10 @@ headless Service. The driver creates an owner-referenced supervisor Deployment with one replica plus Service, proxy CA Secret, and NetworkPolicy resources so agent egress is limited to its paired supervisor pod plus DNS. If the supervisor pod is deleted, the Deployment recreates it. The workload pod -does not mount gateway credentials or the supervisor binary. This topology +does not mount gateway credentials or the supervisor binary. Its proxy CA and +default workspace init containers run as the resolved sandbox UID/GID, disable +privilege escalation, and drop all capabilities. The workload mounts the +generated proxy CA bundle read-only. This topology intentionally omits filesystem/process/binary enforcement, SSH/exec, upload/download, sync, and provider environment injection. diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 66a158cb99..3cca601110 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -3331,17 +3331,19 @@ fn proxy_pod_ca_source_volume_mount() -> serde_json::Value { }) } -fn proxy_pod_ca_tls_volume_mount() -> serde_json::Value { +fn proxy_pod_ca_tls_volume_mount(read_only: bool) -> serde_json::Value { serde_json::json!({ "name": "openshell-proxy-pod-tls", "mountPath": SIDECAR_TLS_MOUNT_PATH, + "readOnly": read_only, }) } fn proxy_pod_ca_init_container( image: &str, image_pull_policy: &str, - sandbox_gid: u32, + run_as_user: u32, + run_as_group: u32, ) -> serde_json::Value { let copy_cmd = format!( "set -eu; \ @@ -3361,16 +3363,18 @@ fn proxy_pod_ca_init_container( "image": image, "command": ["sh", "-c", copy_cmd], "securityContext": { - "runAsUser": 0, - "runAsGroup": sandbox_gid, + "runAsUser": run_as_user, + "runAsGroup": run_as_group, + "runAsNonRoot": true, "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, "capabilities": { "drop": ["ALL"] } }, "volumeMounts": [ proxy_pod_ca_source_volume_mount(), - proxy_pod_ca_tls_volume_mount(), + proxy_pod_ca_tls_volume_mount(false), ] }); if !image_pull_policy.is_empty() { @@ -3493,6 +3497,7 @@ fn apply_supervisor_proxy_pod_topology( init_containers.push(proxy_pod_ca_init_container( params.supervisor_image, params.supervisor_image_pull_policy, + params.sandbox_uid, params.sandbox_gid, )); } @@ -3542,7 +3547,7 @@ fn apply_supervisor_proxy_pod_topology( remove_volume_mount(volume_mounts, SERVICE_ACCOUNT_TOKEN_VOLUME_NAME); remove_volume_mount(volume_mounts, CLIENT_TLS_VOLUME_NAME); remove_volume_mount(volume_mounts, SPIFFE_WORKLOAD_API_VOLUME_NAME); - volume_mounts.push(proxy_pod_ca_tls_volume_mount()); + volume_mounts.push(proxy_pod_ca_tls_volume_mount(true)); } let env = container @@ -3634,7 +3639,9 @@ fn apply_workspace_persistence( pod_template: &mut serde_json::Value, image: &str, image_pull_policy: &str, + sandbox_uid: u32, sandbox_gid: u32, + topology: SupervisorTopology, ) { let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { return; @@ -3711,13 +3718,26 @@ fn apply_workspace_persistence( fi" ); + let security_context = if topology == SupervisorTopology::ProxyPod { + serde_json::json!({ + "runAsUser": sandbox_uid, + "runAsGroup": sandbox_gid, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + } + }) + } else { + serde_json::json!({ + "runAsUser": 0, + }) + }; let mut init_spec = serde_json::json!({ "name": WORKSPACE_INIT_CONTAINER_NAME, "image": image, "command": ["sh", "-c", copy_cmd], - "securityContext": { - "runAsUser": 0, - }, + "securityContext": security_context, "volumeMounts": [{ "name": WORKSPACE_VOLUME_NAME, "mountPath": WORKSPACE_INIT_MOUNT_PATH @@ -4357,7 +4377,9 @@ fn sandbox_template_to_k8s_with_validated_config( &mut result, image, params.image_pull_policy, + params.sandbox_uid, params.sandbox_gid, + params.topology, ); } @@ -4736,7 +4758,7 @@ fn proxy_pod_supervisor_deployment( "mountPath": PROXY_POD_CA_SECRET_MOUNT_PATH, "readOnly": true }, - proxy_pod_ca_tls_volume_mount(), + proxy_pod_ca_tls_volume_mount(false), ] }); if !params.supervisor_image_pull_policy.is_empty() { @@ -7257,6 +7279,13 @@ mod tests { agent["securityContext"]["capabilities"]["drop"], serde_json::json!(["ALL"]) ); + let proxy_tls_mount = agent["volumeMounts"] + .as_array() + .unwrap() + .iter() + .find(|mount| mount["name"] == "openshell-proxy-pod-tls") + .unwrap(); + assert_eq!(proxy_tls_mount["readOnly"], true); let containers = pod_template["spec"]["containers"].as_array().unwrap(); assert_eq!(containers.len(), 1); @@ -7281,10 +7310,23 @@ mod tests { })); let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); - assert!(init_containers.iter().any(|container| { - container["name"] == "openshell-proxy-ca-install" - && container["image"] == "supervisor-image:latest" - })); + let ca_init = init_containers + .iter() + .find(|container| container["name"] == "openshell-proxy-ca-install") + .unwrap(); + assert_eq!(ca_init["image"], "supervisor-image:latest"); + assert_eq!(ca_init["securityContext"]["runAsUser"], 1500); + assert_eq!(ca_init["securityContext"]["runAsGroup"], 1500); + assert_eq!(ca_init["securityContext"]["runAsNonRoot"], true); + assert_eq!( + ca_init["securityContext"]["allowPrivilegeEscalation"], + false + ); + assert_eq!(ca_init["securityContext"]["readOnlyRootFilesystem"], true); + assert_eq!( + ca_init["securityContext"]["capabilities"]["drop"], + serde_json::json!(["ALL"]) + ); assert!( !init_containers .iter() @@ -7294,6 +7336,49 @@ mod tests { assert!(pod_template["spec"].get("affinity").is_none()); } + #[test] + fn proxy_pod_agent_pod_has_no_root_containers() { + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_image: "supervisor-image:latest", + namespace: "agents", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1600, + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + true, + ¶ms, + ); + + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + assert_eq!(init_containers.len(), 2); + for container in containers.iter().chain(init_containers) { + let security_context = &container["securityContext"]; + assert_ne!( + security_context["runAsUser"], 0, + "{} must not run as root", + container["name"] + ); + assert_eq!(security_context["runAsNonRoot"], true); + assert_eq!(security_context["allowPrivilegeEscalation"], false); + assert_eq!( + security_context["capabilities"]["drop"], + serde_json::json!(["ALL"]) + ); + } + } + #[test] fn proxy_pod_topology_supports_preferred_affinity() { let mut spec = serde_json::Map::new(); @@ -7943,7 +8028,9 @@ mod tests { &mut pod_template, "openshell/sandbox:latest", "IfNotPresent", + 1000, // sandbox_uid 1000, // sandbox_gid + SupervisorTopology::Combined, ); // Init container @@ -8003,6 +8090,8 @@ mod tests { "my-custom-image:v2", "IfNotPresent", 1000, + 1000, + SupervisorTopology::Combined, ); let init_image = pod_template["spec"]["initContainers"][0]["image"] @@ -8025,7 +8114,14 @@ mod tests { } }); - apply_workspace_persistence(&mut pod_template, "img:latest", "Always", 1000); + apply_workspace_persistence( + &mut pod_template, + "img:latest", + "Always", + 1000, + 1000, + SupervisorTopology::Combined, + ); let cmd = pod_template["spec"]["initContainers"][0]["command"] .as_array() @@ -8051,6 +8147,37 @@ mod tests { ); } + #[test] + fn workspace_persistence_uses_non_root_init_container_for_proxy_pod() { + let mut pod_template = serde_json::json!({ + "spec": { + "containers": [{ + "name": "agent", + "image": "img:latest" + }] + } + }); + + apply_workspace_persistence( + &mut pod_template, + "img:latest", + "IfNotPresent", + 1500, + 1600, + SupervisorTopology::ProxyPod, + ); + + let security_context = &pod_template["spec"]["initContainers"][0]["securityContext"]; + assert_eq!(security_context["runAsUser"], 1500); + assert_eq!(security_context["runAsGroup"], 1600); + assert_eq!(security_context["runAsNonRoot"], true); + assert_eq!(security_context["allowPrivilegeEscalation"], false); + assert_eq!( + security_context["capabilities"]["drop"], + serde_json::json!(["ALL"]) + ); + } + #[test] fn workspace_persistence_skipped_when_inject_workspace_false() { let params = SandboxPodParams { diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 41ec08942b..d9ef6d32c7 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -61,8 +61,6 @@ rules: - create - delete - get - - list - - watch - apiGroups: - apps resources: @@ -77,9 +75,6 @@ rules: verbs: - create - delete - - get - - list - - watch - apiGroups: - networking.k8s.io resources: @@ -87,8 +82,5 @@ rules: verbs: - create - delete - - get - - list - - watch {{- end }} {{- end }} diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index 5be3f1d9db..01e0df76c3 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -86,8 +86,6 @@ tests: - create - delete - get - - list - - watch - it: grants ReplicaSet get for proxy-pod supervisor token bootstrap template: templates/role.yaml @@ -120,9 +118,6 @@ tests: verbs: - create - delete - - get - - list - - watch - contains: path: rules content: @@ -133,9 +128,6 @@ tests: verbs: - create - delete - - get - - list - - watch - it: omits proxy-pod RBAC in the default combined topology template: templates/role.yaml @@ -151,8 +143,6 @@ tests: - create - delete - get - - list - - watch - notContains: path: rules content: diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index dfcbf8f22a..1fdc11c673 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -45,7 +45,9 @@ pod: |---|---|---|---|---|---| | `combined` | Supervisor install init container | `0` | Not set | Not set | Copies the supervisor binary into the agent container volume. | | `sidecar` | Network init container | `0` | `false` | Drops `ALL`; adds `NET_ADMIN`, `NET_RAW`, `CHOWN`, and `FOWNER` | Installs pod-local nftables rules and prepares shared sidecar state. | -| `proxy-pod` | Proxy CA install init container | `0:sandbox_gid` | `false` | Drops `ALL` | Uses the supervisor utility image to copy proxy CA material into the agent pod TLS volume. | +| `combined` / `sidecar` | Workspace persistence init container | `0` | Not set | Not set | Seeds the default workspace PVC while preserving the existing topology behavior. | +| `proxy-pod` | Proxy CA install init container | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Copies proxy CA material into the agent pod TLS volume with a read-only root filesystem. | +| `proxy-pod` | Workspace persistence init container | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Seeds the default workspace PVC without granting UID 0 to the agent pod. | ## Combined Topology @@ -226,7 +228,9 @@ on `kubernetes.io/hostname` and preserve any affinity supplied by the workload. The agent pod does not mount or execute the OpenShell supervisor. The driver injects standard proxy variables and proxy CA trust directly into the workload -container. Consequently, proxy-pod topology provides network enforcement only: +container. The CA and default workspace init containers run as the same +non-root UID/GID as the workload, and the workload mounts the generated CA +bundle read-only. Consequently, proxy-pod topology provides network enforcement only: OpenShell filesystem policy, process controls, binary identity, SSH/connect, exec, upload/download, file sync, dynamic provider environment injection, and other process-supervisor features are unavailable. The sandbox image's own From 655767a65d3fd675e94c7ffec6f2707875e95773 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Thu, 20 Aug 2026 16:11:21 -0400 Subject: [PATCH 04/25] docs(rfc): add proxy-pod supervisor topology draft RFC Signed-off-by: Russell Bryant --- rfc/proxy-pod-topology-DRAFT.md | 507 ++++++++++++++++++++++++++++++++ 1 file changed, 507 insertions(+) create mode 100644 rfc/proxy-pod-topology-DRAFT.md diff --git a/rfc/proxy-pod-topology-DRAFT.md b/rfc/proxy-pod-topology-DRAFT.md new file mode 100644 index 0000000000..f0d37dc32e --- /dev/null +++ b/rfc/proxy-pod-topology-DRAFT.md @@ -0,0 +1,507 @@ +--- +authors: + - "@TaylorMutch" + - "@russellb" +state: draft +links: + - https://github.com/NVIDIA/OpenShell/pull/2077 - original proxy-pod topology PR from TaylorMutch + - https://github.com/NVIDIA/OpenShell/pull/2074 - kubernetes combined topology + - https://github.com/NVIDIA/OpenShell/pull/2076 - kubernetes sidecar topology + - https://github.com/NVIDIA/OpenShell/pull/2078 - cni-sidecar topology +--- + +# RFC NNNN - Proxy-Pod Supervisor Topology (and OpenShift Enablement) + + + +## Summary + +This RFC proposes `proxy-pod`, a Kubernetes supervisor topology that moves +network enforcement and gateway forwarding out of the sandbox pod entirely and +into a paired, per-sandbox supervisor `Deployment`. The sandbox pod runs the +agent image directly — no supervisor binary, no gateway credentials, no +privileged init container, no shared process namespace. Egress is fenced by two +per-sandbox Kubernetes `NetworkPolicy` objects rather than by pod-local nftables +rules. + +The tradeoff is explicit and large: `proxy-pod` is a **network-only** topology. +Filesystem policy, process and binary identity controls, SSH, `connect`, `exec`, +upload/download, file sync, and dynamic provider environment injection are all +unavailable, because there is no OpenShell supervisor in the workload pod. In +exchange, the sandbox pod's security context reduces to `runAsNonRoot` with all +Linux capabilities dropped, which is the least-privileged sandbox pod any +OpenShell topology produces. + +The RFC also proposes the changes needed to run this topology on OpenShift. Two +are required and are not satisfied by the current implementation: the DNS egress +peers in the generated `NetworkPolicy` are hardcoded to upstream Kubernetes +conventions that do not exist on OpenShift, and the fixed non-root UIDs the +driver assigns are rejected by the `restricted-v2` SCC. The first needs a +configuration surface; the second is satisfied by the built-in `nonroot-v2` SCC +and needs documentation and a gated Helm grant, not a custom SCC. + +## Motivation + +OpenShell's `combined` topology runs the full supervisor inside the agent +container, which requires that container to carry `SYS_ADMIN`, `NET_ADMIN`, +`SYS_PTRACE`, and `SYSLOG`. The `sidecar` topology moves network enforcement to +a dedicated sidecar and drops the agent container to no added capabilities, but +still needs a **privileged network init container** in every sandbox pod to +install the nftables fence. [`cni-sidecar`](./cni-sidecar-topology-DRAFT.md) +removes that init container by pushing rule installation to a node-level CNI +plugin, but it moves the privilege rather than eliminating it: the CNI DaemonSet +runs `privileged` with host-path writes, and the binary-aware sidecar still runs +as UID 0 with `SYS_PTRACE` and `DAC_READ_SEARCH`. + +All three share an assumption: OpenShell's enforcement point lives inside the +sandbox pod, so the pod must be granted whatever privilege that enforcement +requires. Some clusters will not accept that at any level. Multi-tenant +platforms, regulated environments, and clusters with strict admission policy +often permit only the baseline restricted profile for tenant workloads — no +added capabilities, no root containers, no privileged init containers, no +host-path DaemonSets installed on their behalf. On those clusters OpenShell is +currently not deployable at all. + +Such clusters do, however, almost always enforce `NetworkPolicy`, because that +is the tenant-isolation primitive their platform is already built on. If +OpenShell expresses its egress fence as `NetworkPolicy` instead of nftables, the +enforcement moves to machinery the cluster already runs and already trusts, and +the sandbox pod needs no privilege whatsoever. + +The cost is that a supervisor outside the pod cannot supervise processes inside +it. Filesystem policy, binary identity, and the interactive session paths all +depend on the supervisor sharing the workload's namespaces. `proxy-pod` gives +those up deliberately. It is the right choice when the alternative is not a +richer topology but no OpenShell at all. + +OpenShift is the concrete case driving this now. OpenShell's current OpenShift +guidance requires granting sandbox pods the `privileged` SCC and is documented +as experimental and evaluation-only. `cni-sidecar` improves on that but still +needs a custom SCC carrying `SYS_PTRACE` and `DAC_READ_SEARCH` plus +`runAsUser: RunAsAny`. `proxy-pod` needs neither: with the DNS fix proposed +below, it admits under the built-in, unmodified `nonroot-v2` SCC. That makes it +the first OpenShell topology that runs on OpenShift without a bespoke security +grant. + +## Non-goals + +- **Replacing `combined`, `sidecar`, or `cni-sidecar`.** All remain. `combined` + stays the default and the only topology providing the full supervisor + contract. `proxy-pod` is for clusters that cannot accept in-pod privilege. +- **Restoring the removed supervisor features.** Filesystem policy, process and + binary controls, SSH/`connect`, `exec`, upload/download, sync, and dynamic + provider injection are out of scope for this topology by construction. A + RuntimeClass does not restore them. +- **Working without `NetworkPolicy` enforcement.** The topology has no fallback + fence. On a cluster whose CNI ignores `NetworkPolicy`, the generated policies + are declarative only and the workload can bypass the proxy freely. This RFC + proposes failing loudly, not degrading quietly. +- **DNS-level exfiltration control.** The agent pod is permitted UDP/TCP 53 to + cluster DNS so name resolution works. DNS tunnelling is not addressed here. +- **Installing or configuring a CNI.** This RFC consumes whatever + `NetworkPolicy` implementation the cluster already runs. +- **Per-sandbox supervisor autoscaling or sharing.** The pairing is strictly + 1:1. A shared proxy serving many sandboxes is a different design. + +## Proposal + +### Topology overview + +```mermaid +flowchart TB + Sandbox["agents.x-k8s.io Sandbox"] + + subgraph Namespace["Sandbox namespace"] + subgraph AgentPod["Agent pod — role=agent"] + Workload["Agent workload
sandbox image, run directly
runAsNonRoot, drops ALL"] + end + + Deployment["Supervisor Deployment
replicas: 1, owned by Sandbox CR"] + subgraph SupervisorPod["Supervisor pod — role=supervisor"] + Proxy["openshell-supervisor --mode=network
:3128 proxy, :18080 gateway-fwd"] + end + + Service["Headless Service
clusterIP: None"] + CA["Per-sandbox proxy CA Secret"] + EgressNP["NetworkPolicy: agent egress
supervisor ports + DNS only"] + IngressNP["NetworkPolicy: supervisor ingress
paired agent only"] + end + + Gateway["OpenShell Gateway"] + External["External services"] + + Sandbox --> AgentPod + Sandbox --> Deployment + Deployment --> SupervisorPod + AgentPod -->|"HTTP_PROXY / HTTPS_PROXY"| Service + Service --> Proxy + Proxy -->|"gateway forwarding"| Gateway + Proxy -->|"policy-enforced egress"| External + CA -. mounted .- AgentPod + CA -. mounted .- SupervisorPod + EgressNP -. selects .- AgentPod + IngressNP -. selects .- SupervisorPod +``` + +The key structural difference from every other topology: the supervisor is in a +**different pod, and therefore a different network namespace**. There is no +loopback to redirect to and no shared netns to install rules in, so the fence +cannot be nftables. It is `NetworkPolicy`, and that is the entire security +boundary. + +### Per-sandbox resources + +Creating one `proxy-pod` sandbox creates five OpenShell-managed objects +alongside the `Sandbox` CR, all in the sandbox namespace: + +| Object | Name pattern | Purpose | +|---|---|---| +| `Deployment` | `os-sup--` | Runs the network supervisor, 1 replica | +| `Service` | `os-svc--` | Headless; agent's proxy endpoint | +| `Secret` | `os-ca--` | Per-sandbox generated proxy CA cert + key | +| `NetworkPolicy` | `os-eg--` | Agent egress fence | +| `NetworkPolicy` | `os-ing--` | Supervisor ingress restriction | + +Names are `--` to stay within the 63-character +DNS label limit while remaining collision-resistant and human-recognizable. + +The `Deployment` carries a **controlling** `Sandbox` ownerReference; the other +four carry non-controlling ones. Kubernetes garbage collection therefore reclaims +all five when the sandbox is deleted, and the driver additionally deletes them +explicitly on the delete path so teardown does not wait on the GC controller. The +`Deployment` recreates the supervisor pod if it is deleted independently. + +Because the supervisor pod is created by a `Deployment`, its owner chain is +`Pod → ReplicaSet → Deployment → Sandbox` rather than `Pod → Sandbox`. Gateway +ServiceAccount bootstrap must walk that chain to authenticate the supervisor, +validating each link's UID, which is why the topology needs `apps/replicasets: +get` and `apps/deployments: get` in the sandbox `Role`. + +### Privilege model + +| Component | UID | Priv. escalation | Capabilities | Notes | +|---|---|---|---|---| +| Agent workload container | `sandbox_uid:sandbox_gid` | false | drops `ALL` | Runs the sandbox image's own entrypoint. No supervisor. | +| Proxy CA init container | `sandbox_uid:sandbox_gid` | false | drops `ALL`, `readOnlyRootFilesystem` | Builds the CA bundle into an `emptyDir`. | +| Workspace init container | `sandbox_uid:sandbox_gid` | false | drops `ALL` | Seeds the workspace PVC. Non-root, unlike other topologies. | +| Supervisor container | `proxy_uid:sandbox_gid` | false | drops `ALL` | Separate pod. Holds all gateway credentials. | + +No container in either pod runs as root, requests a capability, or needs a +privileged init container, a shared process namespace, or a node-level +DaemonSet. This is the least-privileged configuration OpenShell produces. + +### Credential isolation + +The workload pod receives **no** gateway endpoint, bootstrap token, projected +ServiceAccount token, client TLS identity, or SPIFFE workload socket. It gets +only `HTTP_PROXY`/`HTTPS_PROXY` pointing at the paired Service, `NO_PROXY`, and +a CA trust bundle exposed through the environment variables the common runtimes +read (`SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, `GIT_SSL_CAINFO`, +`NODE_EXTRA_CA_CERTS`, `DENO_CERT`). + +Credential isolation here is structural rather than procedural. The `sidecar` +topology keeps credentials out of the agent container but must defend a shared +control socket with peer-credential checks and one-shot listener semantics. +`proxy-pod` has no such socket: the credential simply is not in the pod, and the +two pods share no namespace, no filesystem, and no IPC. + +One consequence: because credentials are per-supervisor and the CA is generated +per sandbox, a `proxy-pod` sandbox cannot participate in the corporate +upstream-proxy credential feature, which mounts a `user:pass` Secret into the +container performing network supervision. Mounting it into the workload pod +would defeat the purpose. This RFC proposes rejecting that combination at +configuration validation rather than silently mounting it in the wrong place. + +### The NetworkPolicy contract + +Two policies define the fence: + +**Agent egress** (`policyTypes: [Egress]`, selecting `sandbox-role=agent`) permits +exactly two destinations: + +1. Pods labeled `sandbox-role=supervisor` for this sandbox ID, on TCP 3128 and + TCP 18080. +2. Cluster DNS, on UDP 53 and TCP 53. + +Everything else is denied. **This is load-bearing.** `HTTP_PROXY` is only a +convention a workload may ignore; the egress policy is what makes ignoring it +useless. A cluster that does not enforce `NetworkPolicy` provides no fence at +all in this topology, which is why enforcement is a hard prerequisite and not a +recommendation. + +**Supervisor ingress** (`policyTypes: [Ingress]`, selecting +`sandbox-role=supervisor`) accepts only from the paired agent pod on those same +two ports. Supervisor egress is deliberately unrestricted: it must reach the +gateway and the policy-approved internet, and OpenShell policy — not +`NetworkPolicy` — governs where. + +The `sandbox-role` label selectors are scoped by sandbox ID, so two sandboxes in +one namespace cannot reach each other's supervisors. + +### Cluster DNS peers must be configurable + +The current implementation hardcodes the DNS peer as namespace +`kubernetes.io/metadata.name: kube-system` with pod labels `k8s-app: kube-dns` +or `k8s-app: coredns`. That encodes an upstream Kubernetes convention as if it +were a Kubernetes guarantee. It is not. + +On OpenShift 4.x, verified against a live 4.22.6 / OVN-Kubernetes cluster: +`kube-system` contains no DNS pods at all. Cluster DNS runs in namespace +`openshift-dns` as DaemonSet `dns-default`, with pods labeled +`dns.operator.openshift.io/daemonset-dns=default`. The hardcoded selector matches +nothing, so the agent pod's DNS egress falls through to the policy's implicit +deny and **no name resolution works** — including resolving the paired +supervisor's own Service name. The sandbox is inert. + +This RFC proposes a configurable DNS peer list: + +```toml +[openshell.drivers.kubernetes.proxy_pod] +proxy_uid = 1337 +affinity = "disabled" # disabled | preferred | required + +# Cluster DNS peers for the agent egress NetworkPolicy. Defaults to the +# upstream kube-system/kube-dns and kube-system/coredns conventions. +[[openshell.drivers.kubernetes.proxy_pod.dns_peers]] +namespace_labels = { "kubernetes.io/metadata.name" = "openshift-dns" } +pod_labels = { "dns.operator.openshift.io/daemonset-dns" = "default" } +``` + +with the Helm equivalent under `supervisor.proxyPod.dnsPeers`. When unset, the +existing upstream defaults apply, so no behavior changes for current users. Each +entry becomes one `to` peer in the egress rule; multiple entries are additive. + +Configuration is the right shape rather than platform auto-detection: the driver +would otherwise need cluster-type inference and cluster-wide namespace or pod +read permissions it does not currently hold, and operators running NodeLocal +DNSCache or a non-default DNS deployment need the override regardless of +platform. + +### OpenShift SCC model + +OpenShift's `restricted-v2` SCC sets `runAsUser: MustRunAsRange` and +`fsGroup: MustRunAs`, admitting only UIDs inside the namespace's +`openshift.io/sa.scc.uid-range` annotation — on the verification cluster, +`1000000000/10000`. The driver assigns fixed UIDs (`sandbox_uid` default 1000, +`proxy_uid` default 1337), both far outside that range, so `restricted-v2` +rejects both pods. + +The built-in **`nonroot-v2`** SCC resolves this without a custom SCC. It is +`restricted-v2` with `runAsUser: MustRunAsNonRoot` and `fsGroup: RunAsAny`, +keeping `requiredDropCapabilities: [ALL]`, `allowPrivilegeEscalation: false`, +`allowPrivilegedContainer: false`, no host namespaces, and +`seccompProfiles: [runtime/default]`. Its `allowedCapabilities` is +`[NET_BIND_SERVICE]` only, which `proxy-pod` does not use. Its volume allowlist +covers every volume type the topology needs: `emptyDir`, `secret`, `projected`, +`persistentVolumeClaim`, `csi`, and `configMap`. + +`proxy-pod` therefore admits on OpenShift under an unmodified, Red Hat-shipped +SCC: + +```shell +oc adm policy add-scc-to-user nonroot-v2 -z openshell-sandbox -n openshell +``` + +This RFC proposes rendering that grant from the chart behind a gated value +(`sandboxServiceAccount.openshift.nonrootSCC`, default off, so non-OpenShift +installs never reference OpenShift-only APIs), mirroring how `cni-sidecar` +gates its SCC grants. + +The comparison across topologies is the strongest argument for `proxy-pod` on +OpenShift: + +| Topology | OpenShift SCC required | +|---|---| +| `combined` | `privileged` (current documented guidance, evaluation-only) | +| `sidecar` | custom SCC: `RunAsAny` + `SYS_PTRACE` + `DAC_READ_SEARCH` | +| `cni-sidecar` | custom sandbox SCC, plus `privileged` for the CNI DaemonSet | +| `proxy-pod` | built-in `nonroot-v2`, unmodified | + +An alternative worth recording: the driver could omit `runAsUser`/`runAsGroup`/ +`fsGroup` entirely on OpenShift and let SCC admission assign them from the +namespace range, which would admit under stock `restricted-v2` and require no +grant at all. The `proxy_uid != sandbox_uid` constraint exists to keep the +nftables fence from exempting the workload, and `proxy-pod` has no nftables +fence and no shared namespace, so the constraint is not security-relevant here. +This RFC does not propose it yet, because it interacts with workspace PVC +ownership and needs its own validation, but it is the natural follow-up and +would make `proxy-pod` zero-grant on OpenShift. + +### Same-node placement + +`proxy_pod.affinity` controls pairing: `disabled` (default), `preferred`, or +`required`, matching the paired supervisor on `kubernetes.io/hostname` while +preserving any workload-supplied affinity terms. The default is off, which means +every workload byte crosses the pod network to another node. `preferred` is the +better operational default for latency-sensitive agents; `required` risks +unschedulable pairs under node pressure. The default is left at `disabled` in +this RFC but is a reasonable thing for reviewers to push back on. + +### Feature availability + +| Capability | `combined` | `sidecar` | `cni-sidecar` | `proxy-pod` | +|---|---|---|---|---| +| Network endpoint + L7 policy | yes | yes | yes | yes | +| Filesystem policy | yes | partial (Landlock) | partial (Landlock) | **no** | +| Process / binary identity | yes | yes | yes | **no** | +| SSH / `connect` | yes | yes | yes | **no** | +| `exec` | yes | yes | yes | **no** | +| Upload / download / sync | yes | yes | yes | **no** | +| Dynamic provider env injection | yes | yes | yes | **no** | +| Privileged init container | no | **yes** | no | no | +| Added capabilities in sandbox pod | **yes** | no | no | no | +| Requires NetworkPolicy enforcement | no | no | no | **yes** | + +The sandbox image's own entrypoint and command determine what runs. This +topology suits batch and autonomous agent workloads that need policy-enforced +egress and never need an interactive session. + +## Implementation plan + +**Phase 1 — rebase and correctness (done).** Rebase PR #2077 onto current +`main`. Resolve the drift from multi-namespace gateway support (thread namespace +through the supervisor owner-chain walk and the cleanup path) and from the +corporate upstream-proxy feature (reject `proxy-pod` with proxy credential +Secrets at config validation, fail-closed). + +**Phase 2 — pre-OpenShift fixes.** Configurable `dns_peers` with upstream +defaults. Supervisor `Deployment` lifecycle on `stop_sandbox`, which currently +leaves the supervisor running and billable while the sandbox is stopped. Chart +plumbing and unit coverage for both. + +**Phase 3 — OpenShift enablement.** Gated `nonroot-v2` grant in the chart. +Deploy to an OpenShift 4.x / OVN-Kubernetes cluster and validate empirically: +DNS resolves from the agent pod; unproxied egress is denied; proxied egress is +allowed and policy-evaluated; the generated CA is trusted; both pods admit under +`nonroot-v2`; all five resources are reclaimed on delete. Document the results +in `docs/kubernetes/openshift.mdx`. + +**Phase 4 — test strategy.** The branch adds `mise run e2e:kubernetes:proxy-pod`, +but its `PROXY_POD_E2E` flag currently only prints warnings — it gates nothing. +The full Kubernetes e2e suite runs unchanged, and much of it drives sandboxes +through `exec`, SSH, upload, and sync, which this topology removes by design. A +run would fail broadly on absent capabilities and produce no signal about the +fence. `proxy-pod` needs a capability-scoped suite asserting what the topology +actually promises: egress denial, proxied egress, DNS, CA trust, and resource +GC. Until that exists the `test:e2e` gate on this work is unsatisfiable. + +**Phase 5 — graduation.** Ship experimental. Graduate once the scoped suite runs +in CI on at least one policy-enforcing CNI, and the OpenShift path is validated +end to end. + +## Risks + +**Silent loss of enforcement on a non-enforcing CNI.** The highest-severity +risk. If `NetworkPolicy` is not enforced, the generated policies are inert, the +workload can route around the proxy, and everything still *looks* healthy — +pods run, the supervisor is ready, sandboxes report available. There is no +in-band signal. Mitigation should be active rather than documentary: a startup +probe that verifies a denied egress path is actually denied, failing the sandbox +if the fence is not real. Documentation alone is insufficient for a control +whose failure mode is invisible. + +**Feature-set surprise.** An operator selecting `proxy-pod` for its security +properties may not anticipate that `openshell sandbox exec` and `connect` simply +stop working. The gateway should reject those RPCs for `proxy-pod` sandboxes +with an actionable error naming the topology, rather than failing obscurely. + +**Resource multiplication.** Every sandbox becomes two pods plus three +supporting objects. At scale this doubles pod count, doubles scheduling +pressure, and adds five API objects per sandbox. Namespaces with pod quotas will +hit them at half the expected sandbox count. + +**Cross-node data path.** With affinity `disabled`, all workload egress crosses +the pod network. This adds latency to every request and makes the network path a +new failure mode that in-pod topologies do not have. + +**Per-sandbox CA key at rest.** Each sandbox generates a CA cert and private key +stored in a Kubernetes `Secret`. Anyone who can read Secrets in the sandbox +namespace can mint certificates that the workload will trust. The blast radius +is one sandbox, but it is a new key-at-rest surface that other topologies do not +create. + +**DNS as an open egress channel.** UDP/TCP 53 to cluster DNS is permitted and +unfiltered by OpenShell policy, leaving a DNS tunnelling path out of an +otherwise closed pod. + +**Supervisor restart decoupling.** The `Deployment` recreates the supervisor pod +independently of the agent pod. Unlike `sidecar`, where symmetric exit +guarantees a matched pair, an agent pod here can outlive its supervisor and +continue running with all egress denied until the replacement becomes ready. + +## Alternatives + +### Do nothing + +Clusters that permit no in-pod privilege remain unable to run OpenShell. On +OpenShift specifically, the documented path stays `privileged`-SCC and +evaluation-only. + +### Shared proxy for many sandboxes + +One supervisor `Deployment` per namespace instead of per sandbox would cut the +resource multiplication substantially. Rejected: policy is per sandbox, and a +shared proxy would need in-band sandbox attribution on every connection to +enforce the right policy, reintroducing a trust problem that 1:1 pairing avoids +structurally. + +### Sidecar container in the same pod, without the nftables fence + +Keeps one pod and removes the privileged init container, but without a fence the +workload reaches the network directly through the shared namespace and the proxy +becomes advisory. `NetworkPolicy` cannot help, because it cannot distinguish +containers within one pod. The separate pod is what makes the policy fence +expressible. + +### Rely on an admission webhook to inject proxy settings + +Moves configuration out of the driver but does not create a fence, and adds a +cluster-wide mutating webhook — often a harder sell than the workload permissions +it would replace. + +### Custom OpenShift SCC, as `cni-sidecar` uses + +Unnecessary here. `nonroot-v2` already grants exactly what `proxy-pod` needs. +Shipping a custom SCC when a built-in one suffices adds a cluster-scoped object +and an audit burden for no gain. + +### Auto-detect the DNS peers instead of configuring them + +Requires cluster-type inference plus cluster-wide namespace and pod read +permissions the driver does not hold, and still fails for NodeLocal DNSCache and +non-default DNS deployments. Configuration handles every case with no new RBAC. + +## Prior art + +- `combined`, `sidecar` (#2074, #2076) and `cni-sidecar` + ([RFC](./cni-sidecar-topology-DRAFT.md), #2078) — the in-pod topologies this + one departs from. +- Istio and Linkerd sidecar injection with `NetworkPolicy`-backed mesh + isolation: same reliance on the CNI enforcing policy, same + privilege-versus-enforcement tradeoff, and a comparable ambient/sidecar split. +- Kubernetes egress gateways (Cilium, Calico), which likewise centralize + policy-enforced egress outside the workload pod. + +## Open questions + +- Should a startup fence-verification probe be a **requirement** for graduating + `proxy-pod` out of experimental, given that the failure mode of a + non-enforcing CNI is silent? +- On OVN-Kubernetes, does an egress rule whose peer is a `podSelector` match + correctly once the DNS `Service` ClusterIP is DVR-translated to a backend pod + IP, or is a CIDR-based peer needed for the DNS rule specifically? This needs + empirical confirmation on the OpenShift cluster. +- Should `affinity` default to `preferred` rather than `disabled`, given that + the default sends all workload egress across nodes? +- Should the gateway reject `exec`/`connect`/`upload`/`sync` for `proxy-pod` + sandboxes at the RPC boundary with a topology-specific error? +- Should the driver drop explicit `runAsUser`/`runAsGroup`/`fsGroup` on + OpenShift so `proxy-pod` admits under stock `restricted-v2` with no SCC grant + at all, and what does that imply for workspace PVC ownership? +- Is per-sandbox CA generation the right model, or should the CA be issued by + the gateway and distributed, so the private key never rests in a namespace the + operator's tenants may be able to read? From 55e02e9c359cdbb60ae32d05e426191c8ee96a6d Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Thu, 20 Aug 2026 16:28:35 -0400 Subject: [PATCH 05/25] feat(kubernetes): configurable proxy-pod cluster DNS peers The proxy-pod agent egress NetworkPolicy hardcoded its DNS peers as kube-system/k8s-app=kube-dns and kube-system/k8s-app=coredns. That is an upstream Kubernetes convention, not a guarantee. On OpenShift, cluster DNS runs in the openshift-dns namespace with pods labeled dns.operator.openshift.io/daemonset-dns=default, and kube-system holds no DNS pods at all. The hardcoded selector matches nothing, so DNS egress falls through to the policy's implicit deny and the agent pod cannot resolve any name, including its own paired supervisor Service. The sandbox is inert. Add proxy_pod.dns_peers (Helm: supervisor.proxyPod.dnsPeers), a list of namespace/pod label selector pairs, defaulting to the previous upstream behavior so existing deployments are unaffected. Reject an empty peer list at startup, and render no DNS rule at all rather than an empty 'to' array when the list is empty: in NetworkPolicy semantics an empty 'to' matches every destination, so emitting one would silently open DNS-port egress cluster-wide. Signed-off-by: Russell Bryant --- .../openshell-driver-kubernetes/src/config.rs | 127 ++++++++++ .../openshell-driver-kubernetes/src/driver.rs | 220 +++++++++++++++--- crates/openshell-driver-kubernetes/src/lib.rs | 4 +- .../openshell-driver-kubernetes/src/main.rs | 20 +- deploy/helm/openshell/README.md | 1 + .../openshell/templates/gateway-config.yaml | 12 + .../openshell/tests/gateway_config_test.yaml | 48 ++++ deploy/helm/openshell/values.yaml | 12 + 8 files changed, 407 insertions(+), 37 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 2e96a8901e..196a9aa304 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -221,6 +221,61 @@ impl FromStr for ProxyPodAffinity { } } +/// One cluster-DNS peer in the `proxy-pod` agent egress `NetworkPolicy`. +/// +/// Each peer renders as a single `to` entry combining a `namespaceSelector` +/// and a `podSelector`, so both selectors must match the same pod. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct ProxyPodDnsPeer { + /// Labels matched against the namespace hosting the DNS pods. + pub namespace_labels: BTreeMap, + /// Labels matched against the DNS pods themselves. + pub pod_labels: BTreeMap, +} + +impl ProxyPodDnsPeer { + fn new(namespace_label: (&str, &str), pod_label: (&str, &str)) -> Self { + Self { + namespace_labels: std::iter::once(( + namespace_label.0.to_string(), + namespace_label.1.to_string(), + )) + .collect(), + pod_labels: std::iter::once((pod_label.0.to_string(), pod_label.1.to_string())) + .collect(), + } + } + + fn validate(&self, index: usize) -> Result<(), String> { + if self.namespace_labels.is_empty() && self.pod_labels.is_empty() { + return Err(format!( + "proxy_pod.dns_peers[{index}] must set namespace_labels, pod_labels, or both; an \ + empty peer would allow DNS-port egress to every pod in the cluster" + )); + } + Ok(()) + } +} + +/// Upstream Kubernetes conventions for cluster DNS. +/// +/// These are conventions, not guarantees. `OpenShift`, `NodeLocal` `DNSCache`, and +/// custom DNS deployments all place cluster DNS elsewhere and require +/// `proxy_pod.dns_peers` to be set explicitly. +fn default_proxy_pod_dns_peers() -> Vec { + vec![ + ProxyPodDnsPeer::new( + ("kubernetes.io/metadata.name", "kube-system"), + ("k8s-app", "kube-dns"), + ), + ProxyPodDnsPeer::new( + ("kubernetes.io/metadata.name", "kube-system"), + ("k8s-app", "coredns"), + ), + ] +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesProxyPodConfig { @@ -230,6 +285,12 @@ pub struct KubernetesProxyPodConfig { /// Whether same-node placement with the paired supervisor is disabled, /// preferred, or required. pub affinity: ProxyPodAffinity, + /// Cluster DNS peers permitted by the agent egress `NetworkPolicy`. + /// + /// Defaults to the upstream `kube-system` conventions. Clusters that host + /// DNS elsewhere must override this or the agent pod cannot resolve any + /// name, including its own paired supervisor `Service`. + pub dns_peers: Vec, } impl Default for KubernetesProxyPodConfig { @@ -237,6 +298,7 @@ impl Default for KubernetesProxyPodConfig { Self { proxy_uid: DEFAULT_PROXY_UID, affinity: ProxyPodAffinity::Disabled, + dns_peers: default_proxy_pod_dns_peers(), } } } @@ -251,6 +313,25 @@ impl KubernetesProxyPodConfig { } Ok(()) } + + /// Validate the configured DNS peers. + /// + /// An empty list is rejected rather than silently denying DNS: a + /// `proxy-pod` sandbox with no DNS egress cannot resolve its own paired + /// supervisor `Service` and is inert. + pub fn validate_dns_peers(&self) -> Result<(), String> { + if self.dns_peers.is_empty() { + return Err( + "proxy_pod.dns_peers must not be empty; the agent pod needs cluster DNS to \ + resolve its paired supervisor Service" + .to_string(), + ); + } + for (index, peer) in self.dns_peers.iter().enumerate() { + peer.validate(index)?; + } + Ok(()) + } } /// Kubernetes `AppArmor` profile requested for the sandbox agent container. @@ -1037,6 +1118,52 @@ mod tests { assert_eq!(cfg.topology.to_string(), "proxy-pod"); } + #[test] + fn proxy_pod_dns_peers_default_to_kube_system() { + let cfg = KubernetesProxyPodConfig::default(); + assert_eq!(cfg.dns_peers.len(), 2); + cfg.validate_dns_peers().unwrap(); + } + + #[test] + fn proxy_pod_rejects_empty_dns_peers() { + let cfg = KubernetesProxyPodConfig { + dns_peers: Vec::new(), + ..KubernetesProxyPodConfig::default() + }; + let err = cfg.validate_dns_peers().unwrap_err(); + assert!(err.contains("must not be empty"), "{err}"); + } + + #[test] + fn proxy_pod_rejects_a_dns_peer_with_no_selectors() { + let cfg = KubernetesProxyPodConfig { + dns_peers: vec![ProxyPodDnsPeer::default()], + ..KubernetesProxyPodConfig::default() + }; + let err = cfg.validate_dns_peers().unwrap_err(); + assert!(err.contains("dns_peers[0]"), "{err}"); + } + + #[test] + fn serde_override_proxy_pod_dns_peers_nested() { + let cfg: KubernetesComputeConfig = serde_json::from_value(serde_json::json!({ + "proxy_pod": { + "dns_peers": [{ + "namespace_labels": {"kubernetes.io/metadata.name": "openshift-dns"}, + "pod_labels": {"dns.operator.openshift.io/daemonset-dns": "default"} + }] + } + })) + .unwrap(); + assert_eq!(cfg.proxy_pod.dns_peers.len(), 1); + assert_eq!( + cfg.proxy_pod.dns_peers[0].pod_labels["dns.operator.openshift.io/daemonset-dns"], + "default" + ); + cfg.proxy_pod.validate_dns_peers().unwrap(); + } + #[test] fn serde_override_proxy_pod_proxy_uid_nested() { let json = serde_json::json!({ diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 3cca601110..6da4c0d73c 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -7,7 +7,7 @@ use super::AppArmorProfile; use crate::config::{ DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, OperatorNamespaceAllowlist, - ProxyPodAffinity, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, + ProxyPodAffinity, ProxyPodDnsPeer, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, is_dns_1123_label, managed_namespace, validate_managed_namespace_name, }; use futures::{Stream, StreamExt, TryStreamExt}; @@ -486,6 +486,12 @@ impl KubernetesComputeDriver { config .validate_proxy_uid() .map_err(KubernetesDriverError::Precondition)?; + if config.topology == SupervisorTopology::ProxyPod { + config + .proxy_pod + .validate_dns_peers() + .map_err(KubernetesDriverError::Precondition)?; + } config .validate_upstream_proxy_config() .map_err(KubernetesDriverError::Precondition)?; @@ -1426,6 +1432,7 @@ impl KubernetesComputeDriver { proxy_auth_allow_insecure: self.config.proxy_auth_allow_insecure == Some(true), proxy_connect_by_hostname: self.config.proxy_connect_by_hostname == Some(true), proxy_pod_affinity: self.config.proxy_pod.affinity, + proxy_pod_dns_peers: &self.config.proxy_pod.dns_peers, namespace: &self.config.namespace, service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, @@ -3806,6 +3813,7 @@ struct SandboxPodParams<'a> { proxy_auth_allow_insecure: bool, proxy_connect_by_hostname: bool, proxy_pod_affinity: ProxyPodAffinity, + proxy_pod_dns_peers: &'a [ProxyPodDnsPeer], namespace: &'a str, service_account_name: &'a str, sandbox_id: &'a str, @@ -3849,6 +3857,7 @@ impl Default for SandboxPodParams<'_> { proxy_auth_allow_insecure: false, proxy_connect_by_hostname: false, proxy_pod_affinity: ProxyPodAffinity::Disabled, + proxy_pod_dns_peers: &[], namespace: "default", service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, sandbox_id: "", @@ -4885,11 +4894,67 @@ fn proxy_pod_supervisor_deployment( })) } +/// Build the DNS egress rule for the agent pod, if any peers are configured. +/// +/// Every configured peer becomes one `to` entry in the same rule, so the +/// UDP/TCP 53 port list is stated once regardless of peer count. +/// +/// Returns `None` for an empty peer list. This is deliberately fail-closed: a +/// `NetworkPolicy` egress rule with an empty `to` array matches *every* +/// destination, so emitting one here would silently open DNS-port egress to +/// the whole cluster. Omitting the rule denies DNS instead, and +/// `validate_dns_peers` rejects an empty list at startup so a correctly +/// configured driver never reaches this branch. +fn proxy_pod_dns_egress_rule(peers: &[ProxyPodDnsPeer]) -> Option { + if peers.is_empty() { + return None; + } + let to = peers + .iter() + .map(|peer| { + let mut entry = serde_json::Map::new(); + if !peer.namespace_labels.is_empty() { + entry.insert( + "namespaceSelector".to_string(), + serde_json::json!({"matchLabels": peer.namespace_labels}), + ); + } + if !peer.pod_labels.is_empty() { + entry.insert( + "podSelector".to_string(), + serde_json::json!({"matchLabels": peer.pod_labels}), + ); + } + serde_json::Value::Object(entry) + }) + .collect::>(); + Some(serde_json::json!({ + "to": to, + "ports": [ + {"protocol": "UDP", "port": 53}, + {"protocol": "TCP", "port": 53} + ] + })) +} + fn proxy_pod_agent_egress_network_policy( names: &ProxyPodResourceNames, params: &SandboxPodParams<'_>, owner_ref: serde_json::Value, ) -> NetworkPolicy { + let mut egress = vec![serde_json::json!({ + "to": [{ + "podSelector": { + "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR) + } + }], + "ports": [ + {"protocol": "TCP", "port": PROXY_POD_PROXY_PORT}, + {"protocol": "TCP", "port": PROXY_POD_GATEWAY_FORWARD_PORT} + ] + })]; + egress.extend(proxy_pod_dns_egress_rule(params.proxy_pod_dns_peers)); + k8s_object(serde_json::json!({ "apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", @@ -4904,39 +4969,7 @@ fn proxy_pod_agent_egress_network_policy( "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_AGENT) }, "policyTypes": ["Egress"], - "egress": [ - { - "to": [{ - "podSelector": { - "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR) - } - }], - "ports": [ - {"protocol": "TCP", "port": PROXY_POD_PROXY_PORT}, - {"protocol": "TCP", "port": PROXY_POD_GATEWAY_FORWARD_PORT} - ] - }, - { - "to": [{ - "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}, - "podSelector": {"matchLabels": {"k8s-app": "kube-dns"}} - }], - "ports": [ - {"protocol": "UDP", "port": 53}, - {"protocol": "TCP", "port": 53} - ] - }, - { - "to": [{ - "namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "kube-system"}}, - "podSelector": {"matchLabels": {"k8s-app": "coredns"}} - }], - "ports": [ - {"protocol": "UDP", "port": 53}, - {"protocol": "TCP", "port": 53} - ] - } - ] + "egress": egress } })) } @@ -7546,6 +7579,125 @@ mod tests { assert!(err.to_string().contains("proxy-pod")); } + fn dns_egress_rule(policy: &NetworkPolicy) -> Option { + let policy = serde_json::to_value(policy).unwrap(); + policy["spec"]["egress"] + .as_array() + .unwrap() + .iter() + .find(|rule| { + rule["ports"] + .as_array() + .is_some_and(|ports| ports.iter().any(|port| port["port"] == 53)) + }) + .cloned() + } + + fn proxy_pod_egress_policy_with_dns_peers(peers: &[ProxyPodDnsPeer]) -> NetworkPolicy { + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + namespace: "agents", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + proxy_pod_dns_peers: peers, + ..SandboxPodParams::default() + }; + proxy_pod_agent_egress_network_policy( + &proxy_pod_resource_names("example-sandbox"), + ¶ms, + serde_json::json!({}), + ) + } + + #[test] + fn proxy_pod_dns_peers_default_to_upstream_kube_system_conventions() { + let peers = crate::config::KubernetesProxyPodConfig::default().dns_peers; + let rule = dns_egress_rule(&proxy_pod_egress_policy_with_dns_peers(&peers)).unwrap(); + let to = rule["to"].as_array().unwrap(); + + assert_eq!(to.len(), 2); + for entry in to { + assert_eq!( + entry["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], + "kube-system" + ); + } + let apps: Vec<_> = to + .iter() + .map(|entry| entry["podSelector"]["matchLabels"]["k8s-app"].clone()) + .collect(); + assert!(apps.contains(&serde_json::json!("kube-dns"))); + assert!(apps.contains(&serde_json::json!("coredns"))); + } + + /// `OpenShift` hosts cluster DNS in `openshift-dns`, not `kube-system`, and + /// labels the pods with `dns.operator.openshift.io/daemonset-dns=default`. + /// The upstream default matches nothing there, leaving the agent pod unable + /// to resolve even its own paired supervisor Service. + #[test] + fn proxy_pod_dns_peers_render_openshift_selectors() { + let peers = vec![ProxyPodDnsPeer { + namespace_labels: std::iter::once(( + "kubernetes.io/metadata.name".to_string(), + "openshift-dns".to_string(), + )) + .collect(), + pod_labels: std::iter::once(( + "dns.operator.openshift.io/daemonset-dns".to_string(), + "default".to_string(), + )) + .collect(), + }]; + let rule = dns_egress_rule(&proxy_pod_egress_policy_with_dns_peers(&peers)).unwrap(); + let to = rule["to"].as_array().unwrap(); + + assert_eq!(to.len(), 1); + assert_eq!( + to[0]["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], + "openshift-dns" + ); + assert_eq!( + to[0]["podSelector"]["matchLabels"]["dns.operator.openshift.io/daemonset-dns"], + "default" + ); + } + + /// A `NetworkPolicy` egress rule with an empty `to` array matches every + /// destination. Emitting one for an empty peer list would open DNS-port + /// egress cluster-wide, so the rule is omitted entirely instead. + #[test] + fn proxy_pod_empty_dns_peers_omit_the_rule_rather_than_allowing_all() { + let policy = proxy_pod_egress_policy_with_dns_peers(&[]); + assert!(dns_egress_rule(&policy).is_none()); + + let policy = serde_json::to_value(&policy).unwrap(); + let egress = policy["spec"]["egress"].as_array().unwrap(); + assert_eq!(egress.len(), 1); + assert!( + !egress + .iter() + .any(|rule| rule["to"].as_array().is_some_and(Vec::is_empty)) + ); + } + + #[test] + fn proxy_pod_dns_peers_allow_a_namespace_only_peer() { + let peers = vec![ProxyPodDnsPeer { + namespace_labels: std::iter::once(( + "kubernetes.io/metadata.name".to_string(), + "openshift-dns".to_string(), + )) + .collect(), + pod_labels: BTreeMap::new(), + }]; + let rule = dns_egress_rule(&proxy_pod_egress_policy_with_dns_peers(&peers)).unwrap(); + let to = rule["to"].as_array().unwrap(); + + assert_eq!(to.len(), 1); + assert!(to[0].get("namespaceSelector").is_some()); + assert!(to[0].get("podSelector").is_none()); + } + /// Regression test: TLS mount path must match env var paths. /// The volume is mounted at a specific path and the env vars must point to /// files within that same path, otherwise the sandbox will fail to start diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 99a8aa2487..4c1bde1f80 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -8,8 +8,8 @@ pub mod grpc; pub use config::{ AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesProxyPodConfig, - KubernetesSidecarConfig, ManagedSshIngressConfig, ProxyPodAffinity, SupervisorSideloadMethod, - SupervisorTopology, WorkspaceMode, managed_namespace_prefix, + KubernetesSidecarConfig, ManagedSshIngressConfig, ProxyPodAffinity, ProxyPodDnsPeer, + SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index fdd8e2cdd4..d8b355bb98 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -15,7 +15,7 @@ use openshell_driver_kubernetes::{ AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, KubernetesProxyPodConfig, KubernetesSidecarConfig, ManagedSshIngressConfig, ProxyPodAffinity, - SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, + ProxyPodDnsPeer, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, }; #[derive(Parser, Debug)] @@ -176,6 +176,16 @@ struct Args { )] proxy_pod_affinity: ProxyPodAffinity, + /// Cluster DNS peers for the proxy-pod agent egress `NetworkPolicy`, as a + /// JSON array of `{"namespace_labels": {..}, "pod_labels": {..}}` objects. + /// Defaults to the upstream kube-system conventions, which do not match + /// `OpenShift` or `NodeLocal` `DNSCache` deployments. + #[arg( + long = "proxy-pod-dns-peers", + env = "OPENSHELL_K8S_PROXY_POD_DNS_PEERS" + )] + proxy_pod_dns_peers: Option, + #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, @@ -244,6 +254,13 @@ async fn main() -> Result<()> { }) .collect::>>()?; + let proxy_pod_dns_peers = match args.proxy_pod_dns_peers.as_deref() { + Some(raw) => serde_json::from_str::>(raw) + .into_diagnostic() + .map_err(|err| miette::miette!("--proxy-pod-dns-peers must be a JSON array: {err}"))?, + None => KubernetesProxyPodConfig::default().dns_peers, + }; + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { @@ -275,6 +292,7 @@ async fn main() -> Result<()> { proxy_pod: KubernetesProxyPodConfig { proxy_uid: args.proxy_pod_proxy_uid, affinity: args.proxy_pod_affinity, + dns_peers: proxy_pod_dns_peers, }, https_proxy: args.https_proxy, no_proxy: args.no_proxy, diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index a64488b289..3f9c6f0b3d 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -285,6 +285,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | | supervisor.proxyPod.affinity | string | `"disabled"` | Same-node scheduling relationship between the workload pod and its paired proxy supervisor: disabled, preferred, or required. | +| supervisor.proxyPod.dnsPeers | list | `[]` | Cluster DNS peers permitted by the proxy-pod agent egress NetworkPolicy. Each entry sets `namespaceLabels`, `podLabels`, or both. Empty uses the upstream kube-system/kube-dns and kube-system/coredns conventions, which do NOT match OpenShift (cluster DNS runs in `openshift-dns`) or NodeLocal DNSCache. An agent pod with no matching DNS peer cannot resolve its own paired supervisor Service. For OpenShift: dnsPeers: - namespaceLabels: kubernetes.io/metadata.name: openshift-dns podLabels: dns.operator.openshift.io/daemonset-dns: default | | supervisor.proxyPod.proxyUid | int | `1337` | UID for the network supervisor in proxy-pod topology. The configured UID must not match the sandbox UID. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | | supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 6d182ced46..b21ca73b34 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -209,6 +209,18 @@ data: [openshell.drivers.kubernetes.proxy_pod] proxy_uid = {{ .Values.supervisor.proxyPod.proxyUid | default 1337 }} affinity = {{ .Values.supervisor.proxyPod.affinity | default "disabled" | quote }} + {{- range .Values.supervisor.proxyPod.dnsPeers }} + + [[openshell.drivers.kubernetes.proxy_pod.dns_peers]] + {{- with .namespaceLabels }} + {{- $pairs := list }}{{ range $k, $v := . }}{{ $pairs = append $pairs (printf "%q = %q" $k $v) }}{{ end }} + namespace_labels = { {{ join ", " $pairs }} } + {{- end }} + {{- with .podLabels }} + {{- $pairs := list }}{{ range $k, $v := . }}{{ $pairs = append $pairs (printf "%q = %q" $k $v) }}{{ end }} + pod_labels = { {{ join ", " $pairs }} } + {{- end }} + {{- end }} {{- if not $credentialDrivers }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index afa3dd4636..b314bd07d5 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -596,3 +596,51 @@ tests: asserts: - failedTemplate: errorMessage: "certManager.serverIssuerRef.name is set but certManager.enabled is false \u2014 the external server certificate, its Secret mount, and the gateway TLS configuration all require cert-manager to be enabled. Set certManager.enabled=true or remove certManager.serverIssuerRef.name." + + - it: omits proxy-pod dns_peers when none are configured, keeping driver defaults + template: templates/gateway-config.yaml + set: + supervisor.topology: proxy-pod + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'proxy_pod\.dns_peers' + + - it: renders OpenShift cluster DNS peers for proxy-pod topology + template: templates/gateway-config.yaml + set: + supervisor.topology: proxy-pod + supervisor.proxyPod.dnsPeers: + - namespaceLabels: + kubernetes.io/metadata.name: openshift-dns + podLabels: + dns.operator.openshift.io/daemonset-dns: default + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '\[\[openshell\.drivers\.kubernetes\.proxy_pod\.dns_peers\]\]' + - matchRegex: + path: data["gateway.toml"] + pattern: 'namespace_labels = \{ "kubernetes\.io/metadata\.name" = "openshift-dns" \}' + - matchRegex: + path: data["gateway.toml"] + pattern: 'pod_labels = \{ "dns\.operator\.openshift\.io/daemonset-dns" = "default" \}' + + - it: renders multiple proxy-pod dns peers as repeated array-of-tables entries + template: templates/gateway-config.yaml + set: + supervisor.topology: proxy-pod + supervisor.proxyPod.dnsPeers: + - namespaceLabels: + kubernetes.io/metadata.name: openshift-dns + - namespaceLabels: + kubernetes.io/metadata.name: kube-system + podLabels: + k8s-app: node-local-dns + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?s)dns_peers\]\].*dns_peers\]\]' + - matchRegex: + path: data["gateway.toml"] + pattern: 'pod_labels = \{ "k8s-app" = "node-local-dns" \}' diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 95bfc14dbd..aadf9a3937 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -70,6 +70,18 @@ supervisor: # -- Same-node scheduling relationship between the workload pod and its # paired proxy supervisor: disabled, preferred, or required. affinity: disabled + # -- Cluster DNS peers permitted by the proxy-pod agent egress + # NetworkPolicy. Each entry sets `namespaceLabels`, `podLabels`, or both. + # Empty uses the upstream kube-system/kube-dns and kube-system/coredns + # conventions, which do NOT match OpenShift (cluster DNS runs in + # `openshift-dns`) or NodeLocal DNSCache. An agent pod with no matching DNS + # peer cannot resolve its own paired supervisor Service. For OpenShift: + # dnsPeers: + # - namespaceLabels: + # kubernetes.io/metadata.name: openshift-dns + # podLabels: + # dns.operator.openshift.io/daemonset-dns: default + dnsPeers: [] # -- Operator-owned corporate forward proxy for policy-approved TLS egress # from Kubernetes sandboxes. The workload cannot select or override it. From da54eb8527e7dd4b3733a781f1a624fdb47471dc Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Thu, 20 Aug 2026 16:29:28 -0400 Subject: [PATCH 06/25] fix(kubernetes): stop the proxy-pod supervisor when the sandbox stops In proxy-pod topology the network supervisor runs in its own Deployment, so it does not stop when the agent pod does. A stopped sandbox kept its supervisor pod running indefinitely, consuming a pod slot, CPU, and memory for a sandbox the user believes is stopped. Scale the paired Deployment to zero on stop and back to one on start. The scale-down runs only after the workload has actually stopped so a graceful shutdown that needs egress still has it, and scaling failures are logged rather than failing the start/stop RPC. Extract the stop wait loop into wait_for_sandbox_stopped so the scale-down has a single place to hook, and grant the sandbox Role 'patch' on deployments. Signed-off-by: Russell Bryant --- .../openshell-driver-kubernetes/src/driver.rs | 91 +++++++++++++++++-- deploy/helm/openshell/templates/role.yaml | 8 +- .../tests/sandbox_namespace_test.yaml | 1 + 3 files changed, 91 insertions(+), 9 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 6da4c0d73c..c6ec13a7e8 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1663,6 +1663,57 @@ impl KubernetesComputeDriver { Ok(()) } + /// Scale a sandbox's paired supervisor `Deployment`. + /// + /// The supervisor runs in its own `Deployment`, so it does not stop when + /// the agent pod does. Without this, a stopped sandbox keeps consuming a + /// pod slot, CPU, and memory indefinitely. + /// + /// Failures are logged and swallowed. A supervisor that fails to scale down + /// wastes resources but does not break the stop; a supervisor that fails to + /// scale up is retried by the agent pod's connection attempts and surfaces + /// as a normal readiness failure. Neither should fail the caller's + /// start/stop RPC. + async fn scale_proxy_pod_supervisor(&self, sandbox_name: &str, namespace: &str, replicas: u32) { + if self.config.topology != SupervisorTopology::ProxyPod { + return; + } + let names = proxy_pod_resource_names(sandbox_name); + let deployments: Api = Api::namespaced(self.client.clone(), namespace); + let patch = serde_json::json!({"spec": {"replicas": replicas}}); + let result = tokio::time::timeout( + KUBE_API_TIMEOUT, + deployments.patch( + &names.supervisor_deployment, + &PatchParams::apply("openshell-driver-kubernetes").force(), + &Patch::Merge(&patch), + ), + ) + .await; + match result { + Ok(Ok(_)) => info!( + sandbox_name = %sandbox_name, + deployment = %names.supervisor_deployment, + replicas, + "Scaled proxy-pod supervisor Deployment" + ), + Ok(Err(err)) => warn!( + sandbox_name = %sandbox_name, + deployment = %names.supervisor_deployment, + replicas, + error = %err, + "Failed to scale proxy-pod supervisor Deployment" + ), + Err(_elapsed) => warn!( + sandbox_name = %sandbox_name, + deployment = %names.supervisor_deployment, + replicas, + timeout_secs = KUBE_API_TIMEOUT.as_secs(), + "Timed out scaling proxy-pod supervisor Deployment" + ), + } + } + async fn cleanup_proxy_pod_resources(&self, sandbox_name: &str, namespace: &str) { let names = proxy_pod_resource_names(sandbox_name); let secrets: Api = Api::namespaced(self.client.clone(), namespace); @@ -1704,8 +1755,34 @@ impl KubernetesComputeDriver { let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout) = self .patch_sandbox_operating_state(sandbox_id, false) .await?; + let stopped = self + .wait_for_sandbox_stopped( + &agent_sandbox_api, + &kube_name, + &pod_name, + &namespace, + stop_timeout, + ) + .await; + // Scale the paired supervisor down only once the workload has actually + // stopped, so a graceful shutdown that needs egress still has it. + if stopped.is_ok() { + self.scale_proxy_pod_supervisor(&kube_name, &namespace, 0) + .await; + } + stopped + } + + async fn wait_for_sandbox_stopped( + &self, + agent_sandbox_api: &AgentSandboxApi, + kube_name: &str, + pod_name: &str, + namespace: &str, + stop_timeout: Duration, + ) -> Result<(), KubernetesDriverError> { let legacy_pod_api = (agent_sandbox_api.resource.version == SANDBOX_VERSION_V1ALPHA1) - .then(|| Api::::namespaced(self.client.clone(), &namespace)); + .then(|| Api::::namespaced(self.client.clone(), namespace)); let deadline = tokio::time::Instant::now() + stop_timeout; let mut poll_interval = STOP_INITIAL_POLL_INTERVAL; @@ -1720,7 +1797,7 @@ impl KubernetesComputeDriver { let request_timeout = KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)); let object = tokio::time::timeout( request_timeout, - agent_sandbox_api.api.get(&kube_name), + agent_sandbox_api.api.get(kube_name), ) .await .map_err(|_| { @@ -1737,7 +1814,7 @@ impl KubernetesComputeDriver { return Err(KubernetesDriverError::Message(error)); } if let Some(pod_api) = legacy_pod_api.as_ref() - && kubernetes_sandbox_pod_is_gone(pod_api, &pod_name, deadline) + && kubernetes_sandbox_pod_is_gone(pod_api, pod_name, deadline) .await .map_err(KubernetesDriverError::Message)? { @@ -1756,9 +1833,11 @@ impl KubernetesComputeDriver { } pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - self.patch_sandbox_operating_state(sandbox_id, true) - .await - .map(|_| ()) + let (_api, kube_name, _pod_name, namespace, _timeout) = + self.patch_sandbox_operating_state(sandbox_id, true).await?; + self.scale_proxy_pod_supervisor(&kube_name, &namespace, 1) + .await; + Ok(()) } async fn patch_sandbox_operating_state( diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index d9ef6d32c7..6b8bc7c1c0 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -50,9 +50,10 @@ rules: # Service, and one CA Secret per sandbox. All are owner-referenced to the # Sandbox CR for garbage collection. The gateway also reads the generated # ReplicaSet during K8s ServiceAccount bootstrap to verify the supervisor - # pod's Pod -> ReplicaSet -> Deployment -> Sandbox owner chain. These - # permissions are only rendered when the Kubernetes driver is configured for - # proxy-pod topology. + # pod's Pod -> ReplicaSet -> Deployment -> Sandbox owner chain. `patch` on + # deployments scales the paired supervisor to zero when the sandbox stops and + # back to one when it starts. These permissions are only rendered when the + # Kubernetes driver is configured for proxy-pod topology. - apiGroups: - apps resources: @@ -61,6 +62,7 @@ rules: - create - delete - get + - patch - apiGroups: - apps resources: diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index 01e0df76c3..7cc824aca7 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -86,6 +86,7 @@ tests: - create - delete - get + - patch - it: grants ReplicaSet get for proxy-pod supervisor token bootstrap template: templates/role.yaml From c41d714924647d8157efa9d25a9b1c84f4e767fb Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Thu, 20 Aug 2026 16:31:52 -0400 Subject: [PATCH 07/25] feat(helm): gated OpenShift nonroot-v2 SCC grant for sandbox pods The Kubernetes driver assigns explicit non-root UIDs to sandbox and supervisor containers. OpenShift's restricted-v2 SCC uses runAsUser: MustRunAsRange and admits only UIDs inside the namespace's openshift.io/sa.scc.uid-range annotation, so it rejects both pods. The built-in nonroot-v2 SCC resolves this without a custom SCC: it is restricted-v2 with runAsUser: MustRunAsNonRoot and fsGroup: RunAsAny, while keeping requiredDropCapabilities ALL, allowPrivilegeEscalation false, no privileged containers, no host namespaces, and seccomp runtime/default. Its volume allowlist already covers every volume type proxy-pod topology uses. Add sandboxServiceAccount.openshift.nonrootSCC, default false so non-OpenShift installs never reference OpenShift-only APIs. When enabled it renders only a ClusterRole and ClusterRoleBinding granting 'use' on the existing nonroot-v2 SCC; no SecurityContextConstraints object is created. This makes proxy-pod the first OpenShell topology that runs on OpenShift under an unmodified, Red Hat-shipped SCC. Signed-off-by: Russell Bryant --- deploy/helm/openshell/README.md | 1 + .../helm/openshell/templates/sandbox-scc.yaml | 47 +++++++++++++++++++ .../openshell/tests/sandbox_scc_test.yaml | 45 ++++++++++++++++++ deploy/helm/openshell/values.yaml | 10 ++++ 4 files changed, 103 insertions(+) create mode 100644 deploy/helm/openshell/templates/sandbox-scc.yaml create mode 100644 deploy/helm/openshell/tests/sandbox_scc_test.yaml diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 3f9c6f0b3d..2cd123a8ad 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -213,6 +213,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | sandboxServiceAccount.annotations | object | `{}` | Annotations to add to the generated sandbox service account. | | sandboxServiceAccount.create | bool | `true` | Create a service account for sandbox pods. | | sandboxServiceAccount.name | string | `""` | Existing service account name for sandbox pods when sandboxServiceAccount.create is false. | +| sandboxServiceAccount.openshift.nonrootSCC | bool | `false` | Grant the built-in OpenShift `nonroot-v2` SCC to the sandbox ServiceAccount. Required on OpenShift for "proxy-pod" topology: the driver assigns explicit non-root UIDs, which `restricted-v2` rejects because it only admits UIDs inside the namespace's openshift.io/sa.scc.uid-range annotation. No custom SCC is created — `nonroot-v2` ships with OpenShift and already permits exactly what this topology needs, keeping drop-ALL capabilities, no privilege escalation, and no host namespaces. Creates a ClusterRole + ClusterRoleBinding. | | securityContext.allowPrivilegeEscalation | bool | `false` | Whether the gateway container can gain additional privileges. | | securityContext.capabilities.drop | list | `["ALL"]` | Linux capabilities dropped from the gateway container. | | securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. | diff --git a/deploy/helm/openshell/templates/sandbox-scc.yaml b/deploy/helm/openshell/templates/sandbox-scc.yaml new file mode 100644 index 0000000000..8aa6b9f347 --- /dev/null +++ b/deploy/helm/openshell/templates/sandbox-scc.yaml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.sandboxServiceAccount.openshift.nonrootSCC }} +# Grant the built-in OpenShift `nonroot-v2` SCC to the sandbox ServiceAccount. +# +# No SecurityContextConstraints object is created: `nonroot-v2` ships with +# OpenShift and already permits exactly what proxy-pod topology needs. It is +# `restricted-v2` with `runAsUser: MustRunAsNonRoot` and `fsGroup: RunAsAny`, +# which admits the driver's explicit non-root UIDs (restricted-v2 rejects them +# because MustRunAsRange only allows UIDs inside the namespace's +# openshift.io/sa.scc.uid-range annotation). It keeps requiredDropCapabilities +# ALL, allowPrivilegeEscalation false, no privileged containers, no host +# namespaces, and seccomp runtime/default. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "openshell.fullname" . }}-sandbox-nonroot-scc + labels: + {{- include "openshell.labels" . | nindent 4 }} + app.kubernetes.io/component: sandbox +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - nonroot-v2 + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "openshell.fullname" . }}-sandbox-nonroot-scc + labels: + {{- include "openshell.labels" . | nindent 4 }} + app.kubernetes.io/component: sandbox +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "openshell.fullname" . }}-sandbox-nonroot-scc +subjects: + - kind: ServiceAccount + name: {{ include "openshell.sandboxServiceAccountName" . }} + namespace: {{ include "openshell.sandboxNamespace" . }} +{{- end }} diff --git a/deploy/helm/openshell/tests/sandbox_scc_test.yaml b/deploy/helm/openshell/tests/sandbox_scc_test.yaml new file mode 100644 index 0000000000..a6017e75e7 --- /dev/null +++ b/deploy/helm/openshell/tests/sandbox_scc_test.yaml @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +suite: OpenShift sandbox SCC grant +templates: + - templates/sandbox-scc.yaml +tests: + - it: renders nothing by default so non-OpenShift installs never reference OpenShift APIs + asserts: + - hasDocuments: + count: 0 + + - it: grants use of the built-in nonroot-v2 SCC when enabled + set: + sandboxServiceAccount.openshift.nonrootSCC: true + documentIndex: 0 + asserts: + - isKind: + of: ClusterRole + - equal: + path: rules[0].resourceNames[0] + value: nonroot-v2 + - equal: + path: rules[0].verbs[0] + value: use + + - it: binds the SCC grant to the sandbox ServiceAccount + set: + sandboxServiceAccount.openshift.nonrootSCC: true + documentIndex: 1 + asserts: + - isKind: + of: ClusterRoleBinding + - equal: + path: subjects[0].kind + value: ServiceAccount + - equal: + path: subjects[0].name + value: RELEASE-NAME-openshell-sandbox + + - it: creates no SecurityContextConstraints object of its own + set: + sandboxServiceAccount.openshift.nonrootSCC: true + asserts: + - hasDocuments: + count: 2 diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index aadf9a3937..5206daf1c2 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -122,6 +122,16 @@ sandboxServiceAccount: annotations: {} # -- Existing service account name for sandbox pods when sandboxServiceAccount.create is false. name: "" + openshift: + # -- Grant the built-in OpenShift `nonroot-v2` SCC to the sandbox + # ServiceAccount. Required on OpenShift for "proxy-pod" topology: the + # driver assigns explicit non-root UIDs, which `restricted-v2` rejects + # because it only admits UIDs inside the namespace's + # openshift.io/sa.scc.uid-range annotation. No custom SCC is created — + # `nonroot-v2` ships with OpenShift and already permits exactly what this + # topology needs, keeping drop-ALL capabilities, no privilege escalation, + # and no host namespaces. Creates a ClusterRole + ClusterRoleBinding. + nonrootSCC: false # -- Extra annotations to add to the gateway pod. podAnnotations: {} From 8c67e385b505cfef21a5decea7301b5923f52865 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Thu, 20 Aug 2026 18:01:50 -0400 Subject: [PATCH 08/25] fix(kubernetes): make the proxy-pod DNS peer port configurable A NetworkPolicy egress rule whose peer is a podSelector is evaluated against the destination pod after Service address translation, so the rule must carry the DNS pods' container port, not the Service port. Upstream CoreDNS listens on 53, so the two coincide. OpenShift's dns-default listens on 5353 and its Service maps 53 onto it, so a rule allowing port 53 never matches and the agent pod still cannot resolve anything. Verified on OpenShift 4.22 / OVN-Kubernetes: with the correct selectors but port 53, DNS failed both via the Service ClusterIP and via the DNS pod IP directly; with port 5353 it resolves. Add a per-peer 'port' field defaulting to 53, and emit one egress rule per peer rather than one shared rule, since a rule's port list applies to all of its 'to' entries and peers may differ. Signed-off-by: Russell Bryant --- .../openshell-driver-kubernetes/src/config.rs | 28 ++++- .../openshell-driver-kubernetes/src/driver.rs | 102 +++++++++++------- deploy/helm/openshell/README.md | 2 +- .../openshell/templates/gateway-config.yaml | 1 + .../openshell/tests/gateway_config_test.yaml | 8 ++ deploy/helm/openshell/values.yaml | 8 +- 6 files changed, 106 insertions(+), 43 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 196a9aa304..2bb3876c60 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -225,15 +225,37 @@ impl FromStr for ProxyPodAffinity { /// /// Each peer renders as a single `to` entry combining a `namespaceSelector` /// and a `podSelector`, so both selectors must match the same pod. -#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct ProxyPodDnsPeer { /// Labels matched against the namespace hosting the DNS pods. pub namespace_labels: BTreeMap, /// Labels matched against the DNS pods themselves. pub pod_labels: BTreeMap, + /// Port the DNS pods actually listen on. + /// + /// This is the **container** port, not the `Service` port. A + /// `NetworkPolicy` egress rule with a `podSelector` peer is evaluated + /// against the destination pod after `Service` address translation, so a + /// `Service` that maps 53 to a different container port needs that + /// container port here. Upstream `CoreDNS` listens on 53; `OpenShift`'s + /// `dns-default` listens on 5353 and maps 53 to it. + pub port: u16, } +impl Default for ProxyPodDnsPeer { + fn default() -> Self { + Self { + namespace_labels: BTreeMap::new(), + pod_labels: BTreeMap::new(), + port: DEFAULT_DNS_PORT, + } + } +} + +/// Default DNS container port, matching upstream `CoreDNS`/kube-dns. +pub const DEFAULT_DNS_PORT: u16 = 53; + impl ProxyPodDnsPeer { fn new(namespace_label: (&str, &str), pod_label: (&str, &str)) -> Self { Self { @@ -244,10 +266,14 @@ impl ProxyPodDnsPeer { .collect(), pod_labels: std::iter::once((pod_label.0.to_string(), pod_label.1.to_string())) .collect(), + port: DEFAULT_DNS_PORT, } } fn validate(&self, index: usize) -> Result<(), String> { + if self.port == 0 { + return Err(format!("proxy_pod.dns_peers[{index}].port must not be 0")); + } if self.namespace_labels.is_empty() && self.pod_labels.is_empty() { return Err(format!( "proxy_pod.dns_peers[{index}] must set namespace_labels, pod_labels, or both; an \ diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index c6ec13a7e8..559b32afd5 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -4973,22 +4973,26 @@ fn proxy_pod_supervisor_deployment( })) } -/// Build the DNS egress rule for the agent pod, if any peers are configured. +/// Build the DNS egress rules for the agent pod. /// -/// Every configured peer becomes one `to` entry in the same rule, so the -/// UDP/TCP 53 port list is stated once regardless of peer count. +/// Emits one rule per configured peer, because peers may listen on different +/// ports and a `NetworkPolicy` rule applies its port list to every `to` entry +/// in that rule. /// -/// Returns `None` for an empty peer list. This is deliberately fail-closed: a -/// `NetworkPolicy` egress rule with an empty `to` array matches *every* -/// destination, so emitting one here would silently open DNS-port egress to -/// the whole cluster. Omitting the rule denies DNS instead, and +/// `peer.port` is the destination **pod** port. Egress rules with a +/// `podSelector` peer are evaluated after `Service` address translation, so a +/// cluster whose DNS `Service` maps 53 onto a different container port needs +/// that container port configured. Upstream `CoreDNS` listens on 53; +/// `OpenShift`'s `dns-default` listens on 5353 and maps 53 to it. +/// +/// Returns an empty vector for an empty peer list. This is deliberately +/// fail-closed: a `NetworkPolicy` egress rule with an empty `to` array matches +/// *every* destination, so emitting one here would silently open DNS-port +/// egress to the whole cluster. Emitting no rule denies DNS instead, and /// `validate_dns_peers` rejects an empty list at startup so a correctly -/// configured driver never reaches this branch. -fn proxy_pod_dns_egress_rule(peers: &[ProxyPodDnsPeer]) -> Option { - if peers.is_empty() { - return None; - } - let to = peers +/// configured driver never reaches that state. +fn proxy_pod_dns_egress_rules(peers: &[ProxyPodDnsPeer]) -> Vec { + peers .iter() .map(|peer| { let mut entry = serde_json::Map::new(); @@ -5004,16 +5008,15 @@ fn proxy_pod_dns_egress_rule(peers: &[ProxyPodDnsPeer]) -> Option>(); - Some(serde_json::json!({ - "to": to, - "ports": [ - {"protocol": "UDP", "port": 53}, - {"protocol": "TCP", "port": 53} - ] - })) + .collect() } fn proxy_pod_agent_egress_network_policy( @@ -5032,7 +5035,7 @@ fn proxy_pod_agent_egress_network_policy( {"protocol": "TCP", "port": PROXY_POD_GATEWAY_FORWARD_PORT} ] })]; - egress.extend(proxy_pod_dns_egress_rule(params.proxy_pod_dns_peers)); + egress.extend(proxy_pod_dns_egress_rules(params.proxy_pod_dns_peers)); k8s_object(serde_json::json!({ "apiVersion": "networking.k8s.io/v1", @@ -7658,18 +7661,23 @@ mod tests { assert!(err.to_string().contains("proxy-pod")); } - fn dns_egress_rule(policy: &NetworkPolicy) -> Option { + /// Every egress rule except the supervisor rule, which is the one carrying + /// the proxy port. + fn dns_egress_rules(policy: &NetworkPolicy) -> Vec { let policy = serde_json::to_value(policy).unwrap(); policy["spec"]["egress"] .as_array() .unwrap() .iter() - .find(|rule| { - rule["ports"] - .as_array() - .is_some_and(|ports| ports.iter().any(|port| port["port"] == 53)) + .filter(|rule| { + !rule["ports"].as_array().is_some_and(|ports| { + ports + .iter() + .any(|port| port["port"] == i64::from(PROXY_POD_PROXY_PORT)) + }) }) .cloned() + .collect() } fn proxy_pod_egress_policy_with_dns_peers(peers: &[ProxyPodDnsPeer]) -> NetworkPolicy { @@ -7691,20 +7699,22 @@ mod tests { #[test] fn proxy_pod_dns_peers_default_to_upstream_kube_system_conventions() { let peers = crate::config::KubernetesProxyPodConfig::default().dns_peers; - let rule = dns_egress_rule(&proxy_pod_egress_policy_with_dns_peers(&peers)).unwrap(); - let to = rule["to"].as_array().unwrap(); + let rules = dns_egress_rules(&proxy_pod_egress_policy_with_dns_peers(&peers)); - assert_eq!(to.len(), 2); - for entry in to { + assert_eq!(rules.len(), 2); + let mut apps = Vec::new(); + for rule in &rules { + let to = rule["to"].as_array().unwrap(); + assert_eq!(to.len(), 1); assert_eq!( - entry["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], + to[0]["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], "kube-system" ); + for port in rule["ports"].as_array().unwrap() { + assert_eq!(port["port"], 53); + } + apps.push(to[0]["podSelector"]["matchLabels"]["k8s-app"].clone()); } - let apps: Vec<_> = to - .iter() - .map(|entry| entry["podSelector"]["matchLabels"]["k8s-app"].clone()) - .collect(); assert!(apps.contains(&serde_json::json!("kube-dns"))); assert!(apps.contains(&serde_json::json!("coredns"))); } @@ -7726,8 +7736,11 @@ mod tests { "default".to_string(), )) .collect(), + port: 5353, }]; - let rule = dns_egress_rule(&proxy_pod_egress_policy_with_dns_peers(&peers)).unwrap(); + let rule = dns_egress_rules(&proxy_pod_egress_policy_with_dns_peers(&peers)) + .pop() + .unwrap(); let to = rule["to"].as_array().unwrap(); assert_eq!(to.len(), 1); @@ -7739,6 +7752,12 @@ mod tests { to[0]["podSelector"]["matchLabels"]["dns.operator.openshift.io/daemonset-dns"], "default" ); + // OpenShift's dns-default Service maps 53 onto container port 5353. + // Egress rules match the destination pod port, so the rule must carry + // 5353 rather than the Service port. + for port in rule["ports"].as_array().unwrap() { + assert_eq!(port["port"], 5353); + } } /// A `NetworkPolicy` egress rule with an empty `to` array matches every @@ -7747,7 +7766,7 @@ mod tests { #[test] fn proxy_pod_empty_dns_peers_omit_the_rule_rather_than_allowing_all() { let policy = proxy_pod_egress_policy_with_dns_peers(&[]); - assert!(dns_egress_rule(&policy).is_none()); + assert!(dns_egress_rules(&policy).is_empty()); let policy = serde_json::to_value(&policy).unwrap(); let egress = policy["spec"]["egress"].as_array().unwrap(); @@ -7768,8 +7787,11 @@ mod tests { )) .collect(), pod_labels: BTreeMap::new(), + port: 5353, }]; - let rule = dns_egress_rule(&proxy_pod_egress_policy_with_dns_peers(&peers)).unwrap(); + let rule = dns_egress_rules(&proxy_pod_egress_policy_with_dns_peers(&peers)) + .pop() + .unwrap(); let to = rule["to"].as_array().unwrap(); assert_eq!(to.len(), 1); diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 2cd123a8ad..a81ba24689 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -286,7 +286,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | | supervisor.proxyPod.affinity | string | `"disabled"` | Same-node scheduling relationship between the workload pod and its paired proxy supervisor: disabled, preferred, or required. | -| supervisor.proxyPod.dnsPeers | list | `[]` | Cluster DNS peers permitted by the proxy-pod agent egress NetworkPolicy. Each entry sets `namespaceLabels`, `podLabels`, or both. Empty uses the upstream kube-system/kube-dns and kube-system/coredns conventions, which do NOT match OpenShift (cluster DNS runs in `openshift-dns`) or NodeLocal DNSCache. An agent pod with no matching DNS peer cannot resolve its own paired supervisor Service. For OpenShift: dnsPeers: - namespaceLabels: kubernetes.io/metadata.name: openshift-dns podLabels: dns.operator.openshift.io/daemonset-dns: default | +| supervisor.proxyPod.dnsPeers | list | `[]` | Cluster DNS peers permitted by the proxy-pod agent egress NetworkPolicy. Each entry sets `namespaceLabels`, `podLabels`, or both. Empty uses the upstream kube-system/kube-dns and kube-system/coredns conventions, which do NOT match OpenShift (cluster DNS runs in `openshift-dns`) or NodeLocal DNSCache. An agent pod with no matching DNS peer cannot resolve its own paired supervisor Service. `port` is the DNS *pod* port, not the Service port: egress rules with a podSelector match after Service address translation. Upstream CoreDNS listens on 53; OpenShift's dns-default listens on 5353 and maps 53 to it. For OpenShift: dnsPeers: - namespaceLabels: kubernetes.io/metadata.name: openshift-dns podLabels: dns.operator.openshift.io/daemonset-dns: default port: 5353 | | supervisor.proxyPod.proxyUid | int | `1337` | UID for the network supervisor in proxy-pod topology. The configured UID must not match the sandbox UID. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | | supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index b21ca73b34..e06718a0a0 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -220,6 +220,7 @@ data: {{- $pairs := list }}{{ range $k, $v := . }}{{ $pairs = append $pairs (printf "%q = %q" $k $v) }}{{ end }} pod_labels = { {{ join ", " $pairs }} } {{- end }} + port = {{ .port | default 53 }} {{- end }} {{- if not $credentialDrivers }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index b314bd07d5..7a34737cc2 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -615,7 +615,11 @@ tests: kubernetes.io/metadata.name: openshift-dns podLabels: dns.operator.openshift.io/daemonset-dns: default + port: 5353 asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'port = 5353' - matchRegex: path: data["gateway.toml"] pattern: '\[\[openshell\.drivers\.kubernetes\.proxy_pod\.dns_peers\]\]' @@ -638,6 +642,10 @@ tests: podLabels: k8s-app: node-local-dns asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'port = 53' + - matchRegex: path: data["gateway.toml"] pattern: '(?s)dns_peers\]\].*dns_peers\]\]' diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 5206daf1c2..5b2665f0e4 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -75,12 +75,18 @@ supervisor: # Empty uses the upstream kube-system/kube-dns and kube-system/coredns # conventions, which do NOT match OpenShift (cluster DNS runs in # `openshift-dns`) or NodeLocal DNSCache. An agent pod with no matching DNS - # peer cannot resolve its own paired supervisor Service. For OpenShift: + # peer cannot resolve its own paired supervisor Service. + # + # `port` is the DNS *pod* port, not the Service port: egress rules with a + # podSelector match after Service address translation. Upstream CoreDNS + # listens on 53; OpenShift's dns-default listens on 5353 and maps 53 to it. + # For OpenShift: # dnsPeers: # - namespaceLabels: # kubernetes.io/metadata.name: openshift-dns # podLabels: # dns.operator.openshift.io/daemonset-dns: default + # port: 5353 dnsPeers: [] # -- Operator-owned corporate forward proxy for policy-approved TLS egress From 59f6d9c1690169597169a14579a61753c0af937c Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Thu, 20 Aug 2026 18:03:28 -0400 Subject: [PATCH 09/25] docs(rfc): record proxy-pod OpenShift validation results Update the RFC with what a live OpenShift 4.22 / OVN-Kubernetes deployment showed: the DNS peer port mismatch, the measured SCC split between the two pods, and two usability gaps that block adoption -- the user-supplied workload command is silently discarded, and sandboxes never leave Provisioning because nothing opens the supervisor session the Ready transition depends on. Replace the now-answered open question about OVN-Kubernetes service address translation with the questions those findings raise. Signed-off-by: Russell Bryant --- rfc/proxy-pod-topology-DRAFT.md | 134 +++++++++++++++++++++++++++----- 1 file changed, 114 insertions(+), 20 deletions(-) diff --git a/rfc/proxy-pod-topology-DRAFT.md b/rfc/proxy-pod-topology-DRAFT.md index f0d37dc32e..29f58ed6f4 100644 --- a/rfc/proxy-pod-topology-DRAFT.md +++ b/rfc/proxy-pod-topology-DRAFT.md @@ -36,13 +36,20 @@ exchange, the sandbox pod's security context reduces to `runAsNonRoot` with all Linux capabilities dropped, which is the least-privileged sandbox pod any OpenShell topology produces. -The RFC also proposes the changes needed to run this topology on OpenShift. Two -are required and are not satisfied by the current implementation: the DNS egress -peers in the generated `NetworkPolicy` are hardcoded to upstream Kubernetes -conventions that do not exist on OpenShift, and the fixed non-root UIDs the -driver assigns are rejected by the `restricted-v2` SCC. The first needs a -configuration surface; the second is satisfied by the built-in `nonroot-v2` SCC -and needs documentation and a gated Helm grant, not a custom SCC. +The RFC also proposes the changes needed to run this topology on OpenShift, all +validated against a live OpenShift 4.22 / OVN-Kubernetes cluster. Two were +required and unmet by the original implementation: the DNS egress peers in the +generated `NetworkPolicy` are hardcoded to upstream Kubernetes conventions — +both the namespace/pod selectors and the port — that do not hold on OpenShift, +and the driver's explicit non-root proxy UID is rejected by the `restricted-v2` +SCC. The first needs a configuration surface; the second is satisfied by the +built-in `nonroot-v2` SCC and needs a gated Helm grant, not a custom SCC. + +Validation confirmed the security model works as designed on OpenShift — +unproxied egress denied, proxied egress policy-evaluated, resources +garbage-collected — and surfaced two usability gaps that block adoption: the +user-supplied workload command is silently discarded, and sandboxes never leave +the `Provisioning` phase. ## Motivation @@ -257,7 +264,18 @@ nothing, so the agent pod's DNS egress falls through to the policy's implicit deny and **no name resolution works** — including resolving the paired supervisor's own Service name. The sandbox is inert. -This RFC proposes a configurable DNS peer list: +There is a second, subtler mismatch. A `NetworkPolicy` egress rule whose peer +is a `podSelector` is evaluated against the destination **pod** after `Service` +address translation, so its port list must name the DNS pods' *container* port. +Upstream `CoreDNS` listens on 53, so the Service port and container port +coincide and nobody notices. OpenShift's `dns-default` listens on **5353** and +maps 53 onto it, so a rule allowing port 53 matches nothing even with correct +selectors. This was confirmed empirically: with the right selectors but port +53, DNS failed both through the Service ClusterIP and directly against the DNS +pod IP; with 5353 it resolves. + +This RFC therefore proposes a configurable DNS peer list carrying both +selectors and a port: ```toml [openshell.drivers.kubernetes.proxy_pod] @@ -265,12 +283,16 @@ proxy_uid = 1337 affinity = "disabled" # disabled | preferred | required # Cluster DNS peers for the agent egress NetworkPolicy. Defaults to the -# upstream kube-system/kube-dns and kube-system/coredns conventions. +# upstream kube-system/kube-dns and kube-system/coredns conventions on port 53. [[openshell.drivers.kubernetes.proxy_pod.dns_peers]] namespace_labels = { "kubernetes.io/metadata.name" = "openshift-dns" } pod_labels = { "dns.operator.openshift.io/daemonset-dns" = "default" } +port = 5353 ``` +Each peer renders as its own egress rule, because a rule's port list applies to +every `to` entry in that rule and peers may listen on different ports. + with the Helm equivalent under `supervisor.proxyPod.dnsPeers`. When unset, the existing upstream defaults apply, so no behavior changes for current users. Each entry becomes one `to` peer in the egress rule; multiple entries are additive. @@ -306,6 +328,17 @@ SCC: oc adm policy add-scc-to-user nonroot-v2 -z openshell-sandbox -n openshell ``` +Measured on the validation cluster, the two pods land on *different* SCCs, and +only one needs the grant: + +| Pod | Admitted under | UID | Why | +|---|---|---|---| +| Agent | `restricted-v2` | `1000810000` (SCC-assigned) | `sandbox_uid` is optional and was unset, so no explicit UID to reject | +| Supervisor | `nonroot-v2` | `1337` (explicit) | `proxy_pod.proxy_uid` always has a value, which `restricted-v2` rejects | + +Both ran with `capabilities.drop: ["ALL"]`, `allowPrivilegeEscalation: false`, +and `seccompProfile: RuntimeDefault`. + This RFC proposes rendering that grant from the chart behind a gated value (`sandboxServiceAccount.openshift.nonrootSCC`, default off, so non-OpenShift installs never reference OpenShift-only APIs), mirroring how `cni-sidecar` @@ -329,7 +362,8 @@ nftables fence from exempting the workload, and `proxy-pod` has no nftables fence and no shared namespace, so the constraint is not security-relevant here. This RFC does not propose it yet, because it interacts with workspace PVC ownership and needs its own validation, but it is the natural follow-up and -would make `proxy-pod` zero-grant on OpenShift. +would make `proxy-pod` zero-grant on OpenShift. The measurement above is direct +evidence that it would work: the agent pod already takes exactly this path. ### Same-node placement @@ -341,6 +375,43 @@ better operational default for latency-sensitive agents; `required` risks unschedulable pairs under node pressure. The default is left at `disabled` in this RFC but is a reasonable thing for reviewers to push back on. +### Two gaps that block usability + +Cluster validation surfaced two problems that are not OpenShift-specific and +that this RFC treats as required work, not follow-ups. + +**The workload command has nowhere to go.** In `combined` and `sidecar` the +agent container's command is the supervisor binary, and the user's command +reaches the workload through the gateway session. `proxy-pod` has no supervisor +and no session, and `DriverSandboxTemplate` carries no `command`/`args` field at +all, so `openshell sandbox create -- ` is accepted and then silently +discarded. Worse, OpenShell's own sandbox images have `/bin/bash` as their +entrypoint, which under kubelet with no TTY reads EOF and exits 0 immediately — +so the default image produces a `CrashLoopBackOff` with empty logs. Verified: a +`proxy-pod` sandbox on the stock base image crashlooped, and only an image with +a genuinely long-running entrypoint stayed up. + +Options are to add `command`/`args` to `DriverSandboxTemplate` (a proto change +affecting every driver), to accept them through the Kubernetes driver's +`platform_config` passthrough (driver-local, no proto change), or to reject the +combination at the API boundary. At minimum the gateway must not silently +discard a command the user supplied. + +**Sandboxes never reach `Ready`.** The gateway drives the `Ready` transition +from the supervisor session, which the process supervisor in the agent +container opens. `proxy-pod` has no process supervisor, so nothing opens that +session and the sandbox sits in `Provisioning` forever — even though the +Kubernetes `Sandbox` CR reports `Ready`/`DependenciesReady`, both pods are +running, and policy-enforced egress works end to end. Every `Ready`-gated RPC +is then unreachable: `sandbox stop` fails with *"sandbox must be Ready to stop +(current phase: Provisioning)"*, which in turn makes the supervisor scale-down +proposed above unreachable in practice. + +This needs a readiness path that does not assume an in-pod process supervisor — +most naturally the network supervisor reporting readiness for its paired +sandbox once its proxy is serving, since it already holds the gateway +credentials and polls for policy. + ### Feature availability | Capability | `combined` | `sidecar` | `cni-sidecar` | `proxy-pod` | @@ -373,12 +444,27 @@ defaults. Supervisor `Deployment` lifecycle on `stop_sandbox`, which currently leaves the supervisor running and billable while the sandbox is stopped. Chart plumbing and unit coverage for both. -**Phase 3 — OpenShift enablement.** Gated `nonroot-v2` grant in the chart. -Deploy to an OpenShift 4.x / OVN-Kubernetes cluster and validate empirically: -DNS resolves from the agent pod; unproxied egress is denied; proxied egress is -allowed and policy-evaluated; the generated CA is trusted; both pods admit under -`nonroot-v2`; all five resources are reclaimed on delete. Document the results -in `docs/kubernetes/openshift.mdx`. +**Phase 3 — OpenShift enablement (validated).** Gated `nonroot-v2` grant in the +chart, then deployed to OpenShift 4.22.6 / OVN-Kubernetes. Measured results: + +| Check | Result | +|---|---| +| All five per-sandbox resources created | pass | +| Supervisor pod admitted and running | pass, under `nonroot-v2`, UID 1337 | +| Agent pod admitted and running | pass, under stock `restricted-v2`, SCC-assigned UID | +| DNS resolves from the agent pod | pass, only after the 5353 port fix | +| Agent resolves its paired supervisor `Service` | pass | +| Direct egress to the internet denied | pass | +| Direct egress to the gateway denied | pass | +| Egress to supervisor `:3128` / `:18080` allowed | pass | +| Policy-denied host through the proxy | pass, 403 at CONNECT | +| Policy-allowed host through the proxy | pass, HTTP 200 with the generated CA trusted | +| All resources reclaimed on delete | pass | +| Sandbox reaches `Ready` | **fail** — stuck in `Provisioning` | +| `sandbox stop` / `start` | **blocked** by the `Ready` gate | + +The remaining work is documenting the OpenShift path in +`docs/kubernetes/openshift.mdx` and closing the two gaps above. **Phase 4 — test strategy.** The branch adds `mise run e2e:kubernetes:proxy-pod`, but its `PROXY_POD_E2E` flag currently only prints warnings — it gates nothing. @@ -408,6 +494,10 @@ whose failure mode is invisible. properties may not anticipate that `openshell sandbox exec` and `connect` simply stop working. The gateway should reject those RPCs for `proxy-pod` sandboxes with an actionable error naming the topology, rather than failing obscurely. +The observed behavior today is worse than obscure: a working sandbox reports +`Provisioning` indefinitely and a supplied command is discarded without a +warning, so the failure looks like a broken deployment rather than an +intentional topology limit. **Resource multiplication.** Every sandbox becomes two pods plus three supporting objects. At scale this doubles pod count, doubles scheduling @@ -491,10 +581,14 @@ non-default DNS deployments. Configuration handles every case with no new RBAC. - Should a startup fence-verification probe be a **requirement** for graduating `proxy-pod` out of experimental, given that the failure mode of a non-enforcing CNI is silent? -- On OVN-Kubernetes, does an egress rule whose peer is a `podSelector` match - correctly once the DNS `Service` ClusterIP is DVR-translated to a backend pod - IP, or is a CIDR-based peer needed for the DNS rule specifically? This needs - empirical confirmation on the OpenShift cluster. +- Should the network supervisor own the `Ready` transition for its paired + sandbox, or should the gateway derive `Ready` from the `Sandbox` CR conditions + when the topology has no process supervisor? +- Should the workload command reach the container through a new + `DriverSandboxTemplate` field or through the Kubernetes driver's + `platform_config` passthrough? +- Should OpenShell publish a `proxy-pod`-suitable sandbox image with a + long-running entrypoint, given that the current images crashloop here? - Should `affinity` default to `preferred` rather than `disabled`, given that the default sends all workload egress across nodes? - Should the gateway reject `exec`/`connect`/`upload`/`sync` for `proxy-pod` From 1ef2e11aa49a7a73e84a7743dbbe420d5ec14fe9 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 09:44:54 -0400 Subject: [PATCH 10/25] feat(compute): let drivers declare that a sandbox has no supervisor session The gateway forced SandboxPhase::Provisioning unless a ConnectSupervisor session was live. That session is opened only by openshell-supervisor-process and carries only relays -- SSH, exec, port forwarding, file transfer -- so proxy-pod topology, which has no in-sandbox process supervisor, could never reach Ready. Verified on OpenShift: both pods running and policy-enforced egress working end to end, while the sandbox reported Provisioning forever and every Ready-gated RPC, including stop and start, was unreachable. Add SupervisorSessionModel to DriverSandboxStatus. UNSPECIFIED preserves the existing contract, so drivers that never set it are unaffected. The Kubernetes driver reports NONE for proxy-pod and REQUIRED otherwise, and the gateway then derives readiness from the backend conditions alone. Ready must not become a lie in the process. The agent pod gains a wait-for-proxy init container that blocks on its paired supervisor's proxy port, so the pod is not Ready until egress actually works. This also closes a pre-existing ordering gap where the workload could start before the proxy existed and its early egress simply failed. Relay-backed RPCs now fail immediately with an explanation naming the topology instead of waiting out a session timeout that cannot succeed. Signed-off-by: Russell Bryant --- crates/openshell-driver-docker/src/lib.rs | 12 +- .../openshell-driver-kubernetes/src/driver.rs | 226 ++++++++++++++++-- crates/openshell-driver-podman/src/watcher.rs | 7 +- crates/openshell-driver-vm/src/driver.rs | 4 +- crates/openshell-sandbox/src/main.rs | 67 ++++++ crates/openshell-server/src/compute/mod.rs | 126 +++++++++- .../src/supervisor_session.rs | 39 ++- proto/compute_driver.proto | 25 ++ 8 files changed, 475 insertions(+), 31 deletions(-) diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 33acf1a2c6..3bb901dcc5 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -45,11 +45,11 @@ use openshell_core::proto::compute::v1::{ GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, - StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, - WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, - compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, - watch_sandboxes_event, + StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, SupervisorSessionModel, + ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, + WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesRequest, + WatchSandboxesSandboxEvent, compute_driver_server::ComputeDriver, + gateway_listener_requirement::Selector, watch_sandboxes_event, }; use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, @@ -1749,6 +1749,7 @@ fn pending_sandbox_snapshot( namespace: namespace.to_string(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, sandbox_name: sandbox.name.clone(), instance_id: String::new(), agent_fd: String::new(), @@ -3077,6 +3078,7 @@ fn driver_status_from_summary( let (ready, reason, message, deleting) = container_ready_condition(state); DriverSandboxStatus { + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, sandbox_name: summary_container_name(summary).unwrap_or_else(|| sandbox_name.to_string()), instance_id: summary.id.clone().unwrap_or_default(), agent_fd: String::new(), diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 559b32afd5..b605022b49 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -44,9 +44,9 @@ use openshell_core::proto::compute::v1::{ DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, DriverSandbox as Sandbox, DriverSandboxSpec as SandboxSpec, DriverSandboxStatus as SandboxStatus, DriverSandboxTemplate as SandboxTemplate, - GetCapabilitiesResponse, GpuResourceRequirements, WatchSandboxesDeletedEvent, - WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesSandboxEvent, - watch_sandboxes_event, + GetCapabilitiesResponse, GpuResourceRequirements, SupervisorSessionModel, + WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent, + WatchSandboxesSandboxEvent, watch_sandboxes_event, }; use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose}; @@ -1257,7 +1257,9 @@ impl KubernetesComputeDriver { .namespace .clone() .unwrap_or_else(|| self.config.namespace.clone()); - Ok(sandbox_from_object(&ns, obj).ok().map(|(_, s)| s)) + Ok(sandbox_from_object(&ns, obj, self.config.topology) + .ok() + .map(|(_, s)| s)) }, ), Ok(Err(err)) => { @@ -1311,7 +1313,7 @@ impl KubernetesComputeDriver { .namespace .clone() .unwrap_or_else(|| self.config.namespace.clone()); - match sandbox_from_object(&ns, obj) { + match sandbox_from_object(&ns, obj, self.config.topology) { Ok((_, s)) => Some(s), Err(err) => { warn!(object_name = %name, error = %err, "skipping unrecognized Sandbox in list"); @@ -2063,6 +2065,7 @@ impl KubernetesComputeDriver { async fn watch_sandboxes_single_namespace(&self) -> Result { let namespace = self.config.namespace.clone(); + let topology = self.config.topology; let agent_sandbox_api = self .supported_agent_sandbox_api(self.watch_client.clone(), &self.config.namespace) .await?; @@ -2080,7 +2083,7 @@ impl KubernetesComputeDriver { tokio::select! { result = sandbox_stream.try_next() => match result { Ok(Some(Event::Applied(obj))) => { - if let Ok((kube_name, sandbox)) = sandbox_from_object(&namespace, obj) { + if let Ok((kube_name, sandbox)) = sandbox_from_object(&namespace, obj, topology) { update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &kube_name, &sandbox); let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( @@ -2109,7 +2112,7 @@ impl KubernetesComputeDriver { } Ok(Some(Event::Restarted(objs))) => { for obj in objs { - if let Ok((kube_name, sandbox)) = sandbox_from_object(&namespace, obj) { + if let Ok((kube_name, sandbox)) = sandbox_from_object(&namespace, obj, topology) { update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &kube_name, &sandbox); let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( @@ -2174,6 +2177,7 @@ impl KubernetesComputeDriver { } async fn watch_sandboxes_cluster_wide(&self) -> Result { + let topology = self.config.topology; let sandbox_api_version = self .supported_sandbox_api_version(self.watch_client.clone()) .await?; @@ -2192,7 +2196,7 @@ impl KubernetesComputeDriver { Ok(Some(Event::Applied(obj))) => { let ns = obj.metadata.namespace.clone() .unwrap_or_else(|| default_namespace.clone()); - if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { + if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj, topology) { let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } @@ -2221,7 +2225,7 @@ impl KubernetesComputeDriver { for obj in objs { let ns = obj.metadata.namespace.clone() .unwrap_or_else(|| default_namespace.clone()); - if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { + if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj, topology) { let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } @@ -2448,7 +2452,11 @@ fn is_openshell_managed(obj: &DynamicObject) -> bool { /// Returns `Err` in two cases (callers should skip, not fail): /// - The object is not managed by `OpenShell` (missing/wrong `managed-by` label). /// - The object is managed by `OpenShell` but missing required fields (orphan). -fn sandbox_from_object(namespace: &str, obj: DynamicObject) -> Result<(String, Sandbox), String> { +fn sandbox_from_object( + namespace: &str, + obj: DynamicObject, + topology: SupervisorTopology, +) -> Result<(String, Sandbox), String> { let kube_name = obj.metadata.name.clone().unwrap_or_default(); if !is_openshell_managed(&obj) { @@ -2474,7 +2482,7 @@ fn sandbox_from_object(namespace: &str, obj: DynamicObject) -> Result<(String, S .namespace .clone() .unwrap_or_else(|| namespace.to_string()); - let status = status_from_object(&obj); + let status = status_from_object(&obj, topology); Ok(( kube_name, @@ -2659,6 +2667,11 @@ const SANDBOX_ROLE_SUPERVISOR: &str = "supervisor"; const PROXY_POD_PROXY_PORT: u16 = 3128; const PROXY_POD_GATEWAY_FORWARD_PORT: u16 = 18080; const PROXY_POD_GATEWAY_FORWARD_ADDR: &str = "0.0.0.0:18080"; +const PROXY_POD_WAIT_INIT_CONTAINER_NAME: &str = "openshell-wait-for-proxy"; +/// Upper bound on how long the agent pod waits for its paired supervisor. +/// Exceeding it fails the init container, which surfaces as a pod-level error +/// rather than a workload that silently has no egress. +const PROXY_POD_WAIT_TIMEOUT_SECS: u64 = 180; const PROXY_POD_NETWORK_ENFORCEMENT_MODE: &str = "proxy-pod"; const PROXY_POD_CA_SECRET_MOUNT_PATH: &str = "/var/run/openshell-proxy-ca"; const PROXY_POD_CA_CERT_FILE: &str = "openshell-ca.pem"; @@ -3469,6 +3482,39 @@ fn proxy_pod_ca_init_container( init_spec } +fn proxy_pod_wait_for_proxy_init_container( + image: &str, + image_pull_policy: &str, + run_as_user: u32, + run_as_group: u32, + service_dns: &str, +) -> serde_json::Value { + let mut init_spec = serde_json::json!({ + "name": PROXY_POD_WAIT_INIT_CONTAINER_NAME, + "image": image, + "command": [ + SUPERVISOR_IMAGE_BINARY_PATH, + "wait-for-tcp", + format!("{service_dns}:{PROXY_POD_PROXY_PORT}"), + PROXY_POD_WAIT_TIMEOUT_SECS.to_string(), + ], + "securityContext": { + "runAsUser": run_as_user, + "runAsGroup": run_as_group, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": { + "drop": ["ALL"] + } + } + }); + if !image_pull_policy.is_empty() { + init_spec["imagePullPolicy"] = serde_json::json!(image_pull_policy); + } + init_spec +} + fn apply_proxy_pod_affinity( spec: &mut serde_json::Map, sandbox_id: &str, @@ -3586,6 +3632,18 @@ fn apply_supervisor_proxy_pod_topology( params.sandbox_uid, params.sandbox_gid, )); + // Hold the workload until the paired supervisor is accepting proxy + // connections. Without this the workload starts first, its early + // egress fails, and — because the gateway derives readiness for this + // topology from the pod's Ready condition — the sandbox would report + // Ready while it has no egress path at all. + init_containers.push(proxy_pod_wait_for_proxy_init_container( + params.supervisor_image, + params.supervisor_image_pull_policy, + params.sandbox_uid, + params.sandbox_gid, + &service_dns, + )); } let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { @@ -5375,7 +5433,15 @@ fn platform_config_struct(template: &SandboxTemplate, key: &str) -> Option Option { +/// Convert a `Sandbox` CR's status into the driver contract's status. +/// +/// `topology` decides the supervisor-session model reported to the gateway. +/// `proxy-pod` has no in-sandbox process supervisor, so no `ConnectSupervisor` +/// session will ever open; the gateway must derive readiness from the +/// conditions below instead of waiting forever. The agent pod's +/// `wait-for-proxy` init container is what makes that safe: the pod does not +/// become Ready until its paired supervisor is accepting connections. +fn status_from_object(obj: &DynamicObject, topology: SupervisorTopology) -> Option { let status = obj.data.get("status")?; let status_obj = status.as_object()?; @@ -5413,6 +5479,12 @@ fn status_from_object(obj: &DynamicObject) -> Option { .to_string(), conditions, deleting: obj.metadata.deletion_timestamp.is_some(), + supervisor_session_model: match topology { + SupervisorTopology::ProxyPod => SupervisorSessionModel::None as i32, + SupervisorTopology::Combined | SupervisorTopology::Sidecar => { + SupervisorSessionModel::Required as i32 + } + }, }) } @@ -7477,7 +7549,8 @@ mod tests { let containers = pod_template["spec"]["containers"].as_array().unwrap(); let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); - assert_eq!(init_containers.len(), 2); + // CA install, wait-for-proxy, and workspace seed. + assert_eq!(init_containers.len(), 3); for container in containers.iter().chain(init_containers) { let security_context = &container["securityContext"]; assert_ne!( @@ -7696,6 +7769,120 @@ mod tests { ) } + fn sandbox_object_with_conditions(conditions: &[(&str, &str)]) -> DynamicObject { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut obj = DynamicObject::new("sandbox", &resource); + let conditions: Vec<_> = conditions + .iter() + .map(|(kind, status)| serde_json::json!({"type": kind, "status": status})) + .collect(); + obj.data = serde_json::json!({"status": {"conditions": conditions}}); + obj + } + + #[test] + fn proxy_pod_reports_no_supervisor_session_model() { + let obj = sandbox_object_with_conditions(&[("Ready", "True")]); + let status = status_from_object(&obj, SupervisorTopology::ProxyPod).unwrap(); + assert_eq!( + status.supervisor_session_model, + SupervisorSessionModel::None as i32 + ); + } + + #[test] + fn supervisor_topologies_require_a_supervisor_session() { + let obj = sandbox_object_with_conditions(&[("Ready", "True")]); + for topology in [SupervisorTopology::Combined, SupervisorTopology::Sidecar] { + let status = status_from_object(&obj, topology).unwrap(); + assert_eq!( + status.supervisor_session_model, + SupervisorSessionModel::Required as i32, + "{topology}" + ); + } + } + + /// The agent pod must not report Ready before its paired supervisor is + /// serving: the gateway derives readiness from the pod for this topology, + /// so a pod that is up without a proxy would advertise egress it does not + /// have. + #[test] + fn proxy_pod_agent_waits_for_its_paired_supervisor() { + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_image: "supervisor-image:latest", + namespace: "agents", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1500, + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + let wait = init_containers + .iter() + .find(|c| c["name"] == PROXY_POD_WAIT_INIT_CONTAINER_NAME) + .expect("proxy-pod agent pod waits for its supervisor"); + + let command = wait["command"].as_array().unwrap(); + assert_eq!(command[1], "wait-for-tcp"); + let names = proxy_pod_resource_names("example-sandbox"); + let service_dns = proxy_pod_service_dns(&names.service, "agents"); + assert_eq!(command[2], format!("{service_dns}:3128")); + assert_eq!(wait["securityContext"]["runAsNonRoot"], true); + assert_eq!( + wait["securityContext"]["capabilities"]["drop"], + serde_json::json!(["ALL"]) + ); + } + + #[test] + fn other_topologies_have_no_wait_for_proxy_init_container() { + let params = SandboxPodParams { + topology: SupervisorTopology::Sidecar, + supervisor_image: "supervisor-image:latest", + namespace: "agents", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + let init_containers = pod_template["spec"]["initContainers"] + .as_array() + .cloned() + .unwrap_or_default(); + assert!( + !init_containers + .iter() + .any(|c| c["name"] == PROXY_POD_WAIT_INIT_CONTAINER_NAME) + ); + } + #[test] fn proxy_pod_dns_peers_default_to_upstream_kube_system_conventions() { let peers = crate::config::KubernetesProxyPodConfig::default().dns_peers; @@ -9170,7 +9357,8 @@ mod tests { data: serde_json::json!({}), }; - let (kube_name, sandbox) = sandbox_from_object("default", obj).unwrap(); + let (kube_name, sandbox) = + sandbox_from_object("default", obj, SupervisorTopology::Combined).unwrap(); assert_eq!(kube_name, "alpha--work"); assert_eq!(sandbox.name, "work"); assert_eq!(sandbox.workspace, "alpha"); @@ -9199,7 +9387,8 @@ mod tests { data: serde_json::json!({}), }; - let (_, sandbox) = sandbox_from_object("default", obj).unwrap(); + let (_, sandbox) = + sandbox_from_object("default", obj, SupervisorTopology::Combined).unwrap(); assert_eq!(sandbox.name, "work"); assert_eq!(sandbox.workspace, "alpha"); assert_eq!(sandbox.id, "uuid-456"); @@ -9221,7 +9410,7 @@ mod tests { data: serde_json::json!({}), }; - let result = sandbox_from_object("default", obj); + let result = sandbox_from_object("default", obj, SupervisorTopology::Combined); assert!(result.is_err()); assert!(result.unwrap_err().contains("not managed by openshell")); } @@ -9252,7 +9441,8 @@ mod tests { data: serde_json::json!({}), }; - let (_, sandbox) = sandbox_from_object("openshell", obj).unwrap(); + let (_, sandbox) = + sandbox_from_object("openshell", obj, SupervisorTopology::Combined).unwrap(); assert_eq!(sandbox.namespace, "openshell-gw1-team-a"); assert_eq!(sandbox.workspace, "team-a"); } @@ -9277,7 +9467,7 @@ mod tests { data: serde_json::json!({}), }; - let result = sandbox_from_object("default", obj); + let result = sandbox_from_object("default", obj, SupervisorTopology::Combined); assert!(result.is_err()); assert!(result.unwrap_err().contains("missing sandbox workspace")); } diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index 3e98d16271..13a0f3124a 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -13,8 +13,9 @@ use crate::container::{ use futures::Stream; use openshell_core::ComputeDriverError; use openshell_core::proto::compute::v1::{ - DriverCondition, DriverSandbox, DriverSandboxStatus, WatchSandboxesDeletedEvent, - WatchSandboxesEvent, WatchSandboxesSandboxEvent, watch_sandboxes_event, + DriverCondition, DriverSandbox, DriverSandboxStatus, SupervisorSessionModel, + WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesSandboxEvent, + watch_sandboxes_event, }; use std::collections::HashMap; use std::pin::Pin; @@ -348,6 +349,7 @@ fn build_driver_sandbox( namespace: String::new(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, sandbox_name: instance_name, instance_id, agent_fd: String::new(), @@ -650,6 +652,7 @@ mod tests { namespace: String::new(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, sandbox_name: String::new(), instance_id: short_id("container-id-full"), agent_fd: String::new(), diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 13e57f546d..15ca88632f 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -45,7 +45,7 @@ use openshell_core::proto::compute::v1::{ GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + StopSandboxRequest, StopSandboxResponse, SupervisorSessionModel, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, compute_driver_server::ComputeDriver, watch_sandboxes_event, @@ -5406,6 +5406,7 @@ fn sandbox_snapshot(sandbox: &Sandbox, condition: SandboxCondition, deleting: bo namespace: sandbox.namespace.clone(), workspace: sandbox.workspace.clone(), status: Some(SandboxStatus { + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, sandbox_name: sandbox.name.clone(), instance_id: String::new(), agent_fd: String::new(), @@ -5423,6 +5424,7 @@ fn status_with_condition( deleting: bool, ) -> SandboxStatus { SandboxStatus { + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, sandbox_name: snapshot.name.clone(), instance_id: String::new(), agent_fd: String::new(), diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 2a9b77ee02..95af878b9f 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -6,6 +6,7 @@ use std::path::Path; use std::sync::Arc; use std::sync::atomic::AtomicBool; +use std::time::Duration; use clap::Parser; use miette::{IntoDiagnostic, Result}; @@ -34,6 +35,14 @@ const COPY_SELF_SUBCOMMAND: &str = "copy-self"; const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; const VALIDATE_WORKSPACE_SUBCOMMAND: &str = "validate-workspace"; +/// Subcommand that blocks until a TCP endpoint accepts a connection. +/// +/// Used by the `proxy-pod` agent pod's init container to hold the workload +/// until its paired network supervisor is serving. Without it the workload +/// can start before the proxy exists, its early egress fails, and the +/// Kubernetes `Sandbox` reports Ready while no egress path is available. +const WAIT_FOR_TCP_SUBCOMMAND: &str = "wait-for-tcp"; + /// Default `--mode` value: run both supervisor leaves in a single binary. const DEFAULT_MODE: &str = "network,process"; const SIDECAR_STATE_DIR: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT; @@ -506,6 +515,58 @@ fn run_network_init( )) } +/// Block until `addr` accepts a TCP connection, or the timeout elapses. +/// +/// Deliberately dependency-free: this runs in an init container built from +/// the supervisor image, which has no shell networking tools. +fn wait_for_tcp(args: &[String]) -> Result<()> { + let addr = args.first().ok_or_else(|| { + miette::miette!( + "usage: openshell-sandbox {WAIT_FOR_TCP_SUBCOMMAND} [TIMEOUT_SECS]" + ) + })?; + let timeout_secs: u64 = match args.get(1) { + Some(raw) => raw + .parse() + .map_err(|_| miette::miette!("timeout must be a positive integer: {raw}"))?, + None => 180, + }; + + let deadline = std::time::Instant::now() + Duration::from_secs(timeout_secs); + let mut last_error = String::new(); + loop { + // Re-resolve every attempt: the paired supervisor Service may not have + // endpoints yet when the init container first runs. + match std::net::ToSocketAddrs::to_socket_addrs(&addr.as_str()) { + Ok(mut resolved) => { + let mut connected = false; + for socket_addr in &mut resolved { + match std::net::TcpStream::connect_timeout(&socket_addr, Duration::from_secs(5)) + { + Ok(_) => { + connected = true; + break; + } + Err(err) => last_error = err.to_string(), + } + } + if connected { + println!("network supervisor endpoint {addr} is accepting connections"); + return Ok(()); + } + } + Err(err) => last_error = err.to_string(), + } + + if std::time::Instant::now() >= deadline { + return Err(miette::miette!( + "timed out after {timeout_secs}s waiting for network supervisor at {addr}: {last_error}" + )); + } + std::thread::sleep(Duration::from_millis(500)); + } +} + fn main() -> Result<()> { // Handle `copy-self ` before clap so it works without any of the // sandbox flags. Kubernetes init containers invoke this path to seed an @@ -535,6 +596,12 @@ fn main() -> Result<()> { return validate_workspace(&raw_args[2..]); } + // Handle `wait-for-tcp [TIMEOUT_SECS]` before clap. Runs in the + // agent pod's init container, which has none of the supervisor's config. + if raw_args.get(1).map(String::as_str) == Some(WAIT_FOR_TCP_SUBCOMMAND) { + return wait_for_tcp(&raw_args[2..]); + } + let args = Args::parse(); if args.mode.network_init { diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 30a1303bd5..7598a3e3d9 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -40,9 +40,10 @@ use openshell_core::proto::compute::v1::{ GetGatewayListenerRequirementsResponse, GetSandboxRequest, GpuResourceRequirements as DriverGpuResourceRequirements, ListSandboxesRequest, ResourceRequirements as DriverSandboxResourceRequirements, StartSandboxRequest, - StopSandboxRequest, ValidateSandboxCreateRequest, WatchSandboxesEvent, WatchSandboxesRequest, - compute_driver_client::ComputeDriverClient, compute_driver_server::ComputeDriver, - gateway_listener_requirement::Selector, watch_sandboxes_event, + StopSandboxRequest, SupervisorSessionModel, ValidateSandboxCreateRequest, WatchSandboxesEvent, + WatchSandboxesRequest, compute_driver_client::ComputeDriverClient, + compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, + watch_sandboxes_event, }; use openshell_core::proto::{ PlatformEvent, Sandbox, SandboxCondition, SandboxPhase, SandboxSpec, SandboxStatus, @@ -1351,6 +1352,7 @@ impl ComputeRuntime { let sandbox_id = transition.object_id().to_string(); let expected_resource_version = sandbox_resource_version(transition); let session_connected = self.supervisor_sessions.has_session(&sandbox_id); + self.record_supervisor_session_model(&sandbox_id, snapshot); match self .store .update_message_cas::(&sandbox_id, expected_resource_version, |sandbox| { @@ -2768,6 +2770,7 @@ impl ComputeRuntime { existing_phase: SandboxPhase, ) -> Result<(), String> { let session_connected = self.supervisor_sessions.has_session(&incoming.id); + self.record_supervisor_session_model(&incoming.id, &incoming); let sandbox = self .store .update_message_cas::( @@ -2797,6 +2800,17 @@ impl ComputeRuntime { Ok(()) } + /// Track whether this sandbox's topology can ever open a supervisor + /// session, so relay-backed RPCs fail fast with an explanation instead of + /// waiting out a timeout that cannot succeed. + fn record_supervisor_session_model(&self, sandbox_id: &str, snapshot: &DriverSandbox) { + let Some(status) = snapshot.status.as_ref() else { + return; + }; + self.supervisor_sessions + .set_sessionless(sandbox_id, sandbox_has_no_supervisor_session(status)); + } + pub async fn supervisor_session_connected( &self, sandbox_id: &str, @@ -3637,6 +3651,7 @@ fn build_platform_resources_config( fn driver_status_from_public(status: &SandboxStatus) -> DriverSandboxStatus { DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: status.sandbox_name.clone(), instance_id: status.agent_pod.clone(), agent_fd: status.agent_fd.clone(), @@ -3858,12 +3873,27 @@ fn ensure_supervisor_ready_status(status: &mut Option, sandbox_na ); } +/// Whether the driver reports that this sandbox has no in-sandbox process +/// supervisor, and therefore no `ConnectSupervisor` session. +/// +/// Unset and `Required` both preserve the default contract, so a driver that +/// never sets the field behaves exactly as before. +fn sandbox_has_no_supervisor_session(status: &DriverSandboxStatus) -> bool { + status.supervisor_session_model() == SupervisorSessionModel::None +} + /// Compose the public `SandboxPhase` from backend driver state and supervisor session presence. /// /// The readiness decision is a gateway-owned safety invariant: `SandboxPhase::Ready` means /// "usable through this gateway." The driver contract is the extension point for custom backend /// readiness semantics. RFC-0010 lifecycle hooks observe this decision via `post_commit`; they /// do not modify it. +/// +/// Topologies with no in-sandbox process supervisor use that extension point. +/// They report `SupervisorSessionModel::None`, and the gateway then trusts the +/// backend `Ready` condition, because no session will ever arrive. Such a +/// sandbox is usable for policy-enforced network egress but cannot serve a +/// relay, so relay-backed RPCs are rejected rather than left to time out. struct ComposedPhase { phase: SandboxPhase, session_connected: bool, @@ -3873,6 +3903,7 @@ struct ComposedPhase { impl ComposedPhase { fn new(incoming_status: &DriverSandboxStatus, session_connected: bool) -> Self { let backend_phase = derive_phase(Some(incoming_status)); + let sessionless = sandbox_has_no_supervisor_session(incoming_status); // A live supervisor session is a stronger readiness signal than the backend phase. // set_supervisor_session_state may have already promoted the store record to Ready // before this driver snapshot arrived. Keep Ready rather than letting a lagging @@ -3880,13 +3911,18 @@ impl ComposedPhase { let phase = match backend_phase { SandboxPhase::Error | SandboxPhase::Deleting | SandboxPhase::Stopped => backend_phase, _ if session_connected => SandboxPhase::Ready, + // No session will ever arrive for this topology. The driver is + // responsible for withholding its `Ready` condition until the + // out-of-sandbox supervisor is actually serving. + SandboxPhase::Ready if sessionless => SandboxPhase::Ready, _ => SandboxPhase::Provisioning, }; Self { phase, session_connected, backend_ready_without_session: backend_phase == SandboxPhase::Ready - && !session_connected, + && !session_connected + && !sessionless, } } @@ -5367,6 +5403,7 @@ mod tests { fn make_driver_status(condition: DriverCondition) -> DriverSandboxStatus { DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "test".to_string(), instance_id: "test-pod".to_string(), agent_fd: String::new(), @@ -5384,6 +5421,7 @@ mod tests { workspace: "default".to_string(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: name.to_string(), instance_id: format!("{name}-pod"), agent_fd: String::new(), @@ -5531,6 +5569,80 @@ mod tests { assert_eq!(derive_phase(Some(&status)), SandboxPhase::Deleting); } + fn ready_driver_status() -> DriverSandboxStatus { + let mut condition = make_driver_condition("DependenciesReady", "Pod is Ready"); + condition.status = "True".to_string(); + make_driver_status(condition) + } + + #[test] + fn composed_phase_requires_a_session_by_default() { + let status = ready_driver_status(); + assert_eq!(derive_phase(Some(&status)), SandboxPhase::Ready); + + // Unset session model keeps the historical contract: backend Ready is + // not enough, the gateway waits for a supervisor session. + let composed = ComposedPhase::new(&status, false); + assert_eq!(composed.phase, SandboxPhase::Provisioning); + assert!(composed.backend_ready_without_session); + + let composed = ComposedPhase::new(&status, true); + assert_eq!(composed.phase, SandboxPhase::Ready); + } + + #[test] + fn composed_phase_trusts_the_backend_when_no_session_will_ever_arrive() { + let mut status = ready_driver_status(); + status.supervisor_session_model = SupervisorSessionModel::None as i32; + + let composed = ComposedPhase::new(&status, false); + assert_eq!(composed.phase, SandboxPhase::Ready); + // Not "waiting for a supervisor session" -- none is coming, so the + // sandbox must not advertise that it is still settling. + assert!(!composed.backend_ready_without_session); + } + + #[test] + fn sessionless_sandboxes_are_not_ready_until_the_backend_says_so() { + let mut status = make_driver_status(make_driver_condition( + "DependenciesNotReady", + "Pod exists with phase: Pending", + )); + status.supervisor_session_model = SupervisorSessionModel::None as i32; + + // The driver withholds its Ready condition until the paired supervisor + // is serving, so the gateway must not promote this to Ready. + assert_eq!( + ComposedPhase::new(&status, false).phase, + SandboxPhase::Provisioning + ); + } + + #[test] + fn sessionless_model_does_not_override_terminal_backend_phases() { + for (reason, expected) in [ + ("Failed", SandboxPhase::Error), + ("Suspended", SandboxPhase::Stopped), + ] { + let mut status = if reason == "Suspended" { + let mut status = make_driver_status(make_driver_condition("Suspended", "stopped")); + status.conditions[0].r#type = "Suspended".to_string(); + status.conditions[0].status = "True".to_string(); + status + } else { + let mut status = make_driver_status(make_driver_condition(reason, "failed")); + status.conditions[0].status = "False".to_string(); + status + }; + status.supervisor_session_model = SupervisorSessionModel::None as i32; + assert_eq!( + ComposedPhase::new(&status, false).phase, + expected, + "{reason}" + ); + } + } + #[test] fn derive_phase_returns_provisioning_for_transient_conditions() { let transient_conditions = [ @@ -6559,6 +6671,7 @@ mod tests { namespace: "default".to_string(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "sandbox-a".to_string(), instance_id: "agent-pod".to_string(), agent_fd: String::new(), @@ -7887,6 +8000,7 @@ mod tests { fn make_ready_driver_status() -> DriverSandboxStatus { DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "test".to_string(), instance_id: "test-pod".to_string(), agent_fd: String::new(), @@ -7904,6 +8018,7 @@ mod tests { fn make_deleting_driver_status() -> DriverSandboxStatus { DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "test".to_string(), instance_id: "test-pod".to_string(), agent_fd: String::new(), @@ -8181,6 +8296,7 @@ mod tests { namespace: "default".to_string(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "sandbox-a".to_string(), instance_id: "agent-pod".to_string(), agent_fd: String::new(), @@ -8202,6 +8318,7 @@ mod tests { namespace: "default".to_string(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "sandbox-a".to_string(), instance_id: "agent-pod".to_string(), agent_fd: String::new(), @@ -8415,6 +8532,7 @@ mod tests { namespace: "default".to_string(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "sandbox-a".to_string(), instance_id: "agent-pod".to_string(), agent_fd: String::new(), diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index fbff0e276c..9fe51c2860 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::pin::Pin; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -72,6 +72,10 @@ pub struct SupervisorSessionRegistry { sessions: Mutex>, /// `channel_id` -> oneshot sender for the reverse CONNECT stream. pending_relays: Mutex>, + /// Sandboxes whose topology has no in-sandbox process supervisor, and so + /// will never register a session. Waiting for one is pointless, and the + /// caller deserves to know why rather than watching a timeout elapse. + sessionless: Mutex>, } struct PendingRelay { @@ -182,6 +186,17 @@ impl SupervisorSessionRegistry { sandbox_id: &str, timeout: Duration, ) -> Result, Status> { + // Topologies without an in-sandbox process supervisor never register a + // session. Fail immediately with an actionable message instead of + // burning the caller's timeout on a wait that cannot succeed. + if self.is_sessionless(sandbox_id) { + return Err(Status::failed_precondition( + "this sandbox runs a topology with no in-sandbox supervisor, so SSH, exec, \ + port forwarding, and file transfer are unavailable; use the `combined` or \ + `sidecar` topology when those are required", + )); + } + let deadline = Instant::now() + timeout; let mut backoff = SESSION_WAIT_INITIAL_BACKOFF; @@ -209,6 +224,28 @@ impl SupervisorSessionRegistry { self.sessions.lock().unwrap().contains_key(sandbox_id) } + /// Record whether a sandbox's topology can ever open a supervisor session. + /// + /// Driven by the compute driver's reported `SupervisorSessionModel`, so it + /// re-establishes itself from the next driver snapshot after a gateway + /// restart. + pub fn set_sessionless(&self, sandbox_id: &str, sessionless: bool) { + let mut set = self.sessionless.lock().unwrap(); + if sessionless { + set.insert(sandbox_id.to_string()); + } else { + set.remove(sandbox_id); + } + } + + pub fn is_sessionless(&self, sandbox_id: &str) -> bool { + self.sessionless.lock().unwrap().contains(sandbox_id) + } + + pub fn forget_sessionless(&self, sandbox_id: &str) { + self.sessionless.lock().unwrap().remove(sandbox_id); + } + pub fn is_current_session(&self, sandbox_id: &str, session_id: &str) -> bool { self.sessions .lock() diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index afa93f1b18..eb8edd79b7 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -232,6 +232,31 @@ message DriverSandboxStatus { repeated DriverCondition conditions = 5; // True when the compute platform has begun deleting this sandbox. bool deleting = 6; + // How the gateway should decide that this sandbox is Ready. + // + // Unset preserves the default contract: the gateway requires a live + // supervisor session before reporting Ready. Drivers whose topology has no + // in-sandbox process supervisor report SUPERVISOR_SESSION_MODEL_NONE so the + // gateway derives Ready from `conditions` alone. + SupervisorSessionModel supervisor_session_model = 7; +} + +// Whether a sandbox has an in-sandbox process supervisor that opens a +// `ConnectSupervisor` session with the gateway. +// +// The session is the transport for relays -- SSH, exec, port forwarding, and +// file transfer -- so this also tells the gateway which RPCs the sandbox can +// serve. A sandbox reporting NONE is reachable for policy-enforced network +// egress but cannot accept a relay. +enum SupervisorSessionModel { + // Default contract: a supervisor session is required for Ready and relays + // are expected to work. + SUPERVISOR_SESSION_MODEL_UNSPECIFIED = 0; + // A supervisor session is required before the sandbox is Ready. + SUPERVISOR_SESSION_MODEL_REQUIRED = 1; + // No supervisor session exists. Readiness comes from `conditions`, and + // relay-backed RPCs are unavailable for this sandbox. + SUPERVISOR_SESSION_MODEL_NONE = 2; } // Raw compute-platform condition. From 028801ddd765c20f2f15a855846096f786ef5858 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 09:46:44 -0400 Subject: [PATCH 11/25] feat(kubernetes): workload entrypoint override for proxy-pod topology proxy-pod runs the sandbox image directly, with no supervisor to launch a workload, so the container needs an entrypoint that stays running. OpenShell's own sandbox images use an interactive shell entrypoint, which reads EOF under kubelet and exits 0, leaving the pod in CrashLoopBackOff with empty logs. Add containers.agent.command and containers.agent.args to the Kubernetes driver_config passthrough, alongside the existing resources and volume_mounts. This needs no public API change: the initial command supplied to 'sandbox create' is delivered over the supervisor session, which this topology does not have. Reject the fields in combined and sidecar topology, where the driver replaces the container command with the supervisor binary and an override would be accepted and then silently dropped. Signed-off-by: Russell Bryant --- .../openshell-driver-kubernetes/src/driver.rs | 70 ++++++++++++++++++- docs/kubernetes/topology.mdx | 11 +++ docs/reference/sandbox-compute-drivers.mdx | 14 ++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index b605022b49..4a33a013ed 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -196,6 +196,16 @@ struct KubernetesDriverContainersConfig { struct KubernetesContainerDriverConfig { resources: KubernetesContainerResourceConfig, volume_mounts: Vec, + /// Entrypoint override for the workload container. + /// + /// Only meaningful in `proxy-pod` topology, where the sandbox image runs + /// directly. `combined` and `sidecar` replace the container command with + /// the supervisor binary, so an override there would be silently ignored + /// and is rejected instead. + command: Vec, + /// Arguments for `command`, or for the image entrypoint when `command` is + /// not set. + args: Vec, } #[derive(Debug, Clone, Default, Deserialize)] @@ -1012,14 +1022,16 @@ impl KubernetesComputeDriver { &self, sandbox: &Sandbox, ) -> Result { - kubernetes_driver_config_for_spec( + let config = kubernetes_driver_config_for_spec( sandbox.spec.as_ref(), self.config.provider_spiffe_enabled().then_some( self.config .provider_spiffe_workload_api_socket_path .as_str(), ), - ) + )?; + validate_agent_command_for_topology(&config, self.config.topology)?; + Ok(config) } fn agent_sandbox_api( @@ -4043,6 +4055,30 @@ fn spec_pod_env(spec: Option<&SandboxSpec>) -> std::collections::HashMap Result<(), String> { + let agent = &config.containers.agent; + if agent.command.is_empty() && agent.args.is_empty() { + return Ok(()); + } + if topology == SupervisorTopology::ProxyPod { + return Ok(()); + } + Err(format!( + "containers.agent.command and containers.agent.args are only supported in \"proxy-pod\" \ + topology; {topology} topology runs the OpenShell supervisor as the container entrypoint \ + and would ignore them" + )) +} + fn kubernetes_driver_config_for_spec( spec: Option<&SandboxSpec>, provider_spiffe_workload_api_socket_path: Option<&str>, @@ -4397,6 +4433,18 @@ fn sandbox_template_to_k8s_with_validated_config( container.insert("resources".to_string(), resources); } apply_agent_driver_resources(&mut container, &driver_config.containers.agent.resources); + if params.topology == SupervisorTopology::ProxyPod { + let agent_config = &driver_config.containers.agent; + if !agent_config.command.is_empty() { + container.insert( + "command".to_string(), + serde_json::json!(agent_config.command), + ); + } + if !agent_config.args.is_empty() { + container.insert("args".to_string(), serde_json::json!(agent_config.args)); + } + } spec.insert( "containers".to_string(), serde_json::Value::Array(vec![serde_json::Value::Object(container)]), @@ -7784,6 +7832,24 @@ mod tests { obj } + #[test] + fn agent_command_override_is_rejected_outside_proxy_pod() { + let config = KubernetesSandboxDriverConfig { + containers: KubernetesDriverContainersConfig { + agent: KubernetesContainerDriverConfig { + command: vec!["sleep".to_string(), "infinity".to_string()], + ..KubernetesContainerDriverConfig::default() + }, + }, + ..KubernetesSandboxDriverConfig::default() + }; + + validate_agent_command_for_topology(&config, SupervisorTopology::ProxyPod).unwrap(); + let err = + validate_agent_command_for_topology(&config, SupervisorTopology::Combined).unwrap_err(); + assert!(err.contains("proxy-pod"), "{err}"); + } + #[test] fn proxy_pod_reports_no_supervisor_session_model() { let obj = sandbox_object_with_conditions(&[("Ready", "True")]); diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index 1fdc11c673..7c5f221fa2 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -226,6 +226,17 @@ Same-node scheduling is disabled by default. Set `proxy_pod.affinity` (or Helm `required` for hard same-node placement. Both modes match the paired supervisor on `kubernetes.io/hostname` and preserve any affinity supplied by the workload. +Because the sandbox image runs directly with no supervisor to launch a +workload, the image needs an entrypoint that stays running. OpenShell's own +sandbox images use an interactive shell entrypoint, which exits immediately +under Kubernetes and leaves the pod in `CrashLoopBackOff`. Either use an image +whose entrypoint is long-running, or set an explicit command: + +```shell +openshell sandbox create --name batch \ + --driver-config-json '{"kubernetes":{"containers":{"agent":{"command":["python","/app/agent.py"]}}}}' +``` + The agent pod does not mount or execute the OpenShell supervisor. The driver injects standard proxy variables and proxy CA trust directly into the workload container. The CA and default workspace init containers run as the same diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 6bda8740c1..68eafc6652 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -128,6 +128,20 @@ It overrides the gateway's configured default runtime class for that sandbox, while a typed `SandboxTemplate.runtime_class_name` value from the API still takes precedence. +In `proxy-pod` topology the sandbox image runs directly, with no supervisor to +launch a workload, so the container needs an entrypoint that stays running. +Set `containers.agent.command` and `containers.agent.args` when the image's own +entrypoint exits immediately: + +```shell +openshell sandbox create --name batch \ + --driver-config-json '{"kubernetes":{"containers":{"agent":{"command":["python","/app/agent.py"]}}}}' +``` + +These fields apply only to `proxy-pod`. The `combined` and `sidecar` topologies +run the OpenShell supervisor as the container entrypoint, so setting them there +is rejected rather than silently ignored. + Docker and Podman report the address through which their sandboxes can reach the gateway. If the primary listener covers that address, the gateway reuses it and sandbox JWT authentication restricts the supervisor to its callback RPC From 6ce0a74cc441ce703dcd2cea0eed0feba553a688 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 14:42:11 -0400 Subject: [PATCH 12/25] fix(kubernetes): derive proxy-pod resource names from the sandbox name Per-sandbox proxy-pod resources are named from the sandbox name, but the stop, start, and delete paths passed the Sandbox CR name. The two differ: a CR is named --, so a sandbox named 'rdy' has CR 'default--rdy' and Deployment 'os-sup-rdy-'. The scale-down on stop therefore patched a Deployment that does not exist and silently did nothing, leaving the supervisor running for a stopped sandbox -- the exact problem the scaling was added to fix. Delete was affected too, but owner-reference garbage collection reclaimed the resources anyway and hid it. Read the sandbox name from the CR's sandbox-name label at both sites, and fall back to owner-reference GC with a warning if the label is missing. Caught by cluster testing; the unit tests passed throughout because they never exercised the CR-name-to-resource-name path. Signed-off-by: Russell Bryant --- .../openshell-driver-kubernetes/src/driver.rs | 164 +++++++++++------- 1 file changed, 106 insertions(+), 58 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 4a33a013ed..cbfd4fc75f 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1766,7 +1766,7 @@ impl KubernetesComputeDriver { } pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout) = self + let (agent_sandbox_api, kube_name, sandbox_name, pod_name, namespace, stop_timeout) = self .patch_sandbox_operating_state(sandbox_id, false) .await?; let stopped = self @@ -1780,8 +1780,10 @@ impl KubernetesComputeDriver { .await; // Scale the paired supervisor down only once the workload has actually // stopped, so a graceful shutdown that needs egress still has it. - if stopped.is_ok() { - self.scale_proxy_pod_supervisor(&kube_name, &namespace, 0) + if stopped.is_ok() + && let Some(sandbox_name) = sandbox_name.as_deref() + { + self.scale_proxy_pod_supervisor(sandbox_name, &namespace, 0) .await; } stopped @@ -1847,10 +1849,12 @@ impl KubernetesComputeDriver { } pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let (_api, kube_name, _pod_name, namespace, _timeout) = + let (_api, _kube_name, sandbox_name, _pod_name, namespace, _timeout) = self.patch_sandbox_operating_state(sandbox_id, true).await?; - self.scale_proxy_pod_supervisor(&kube_name, &namespace, 1) - .await; + if let Some(sandbox_name) = sandbox_name.as_deref() { + self.scale_proxy_pod_supervisor(sandbox_name, &namespace, 1) + .await; + } Ok(()) } @@ -1858,7 +1862,17 @@ impl KubernetesComputeDriver { &self, sandbox_id: &str, running: bool, - ) -> Result<(AgentSandboxApi, String, String, String, Duration), KubernetesDriverError> { + ) -> Result< + ( + AgentSandboxApi, + String, + Option, + String, + String, + Duration, + ), + KubernetesDriverError, + > { let lookup_api = self .supported_sandbox_api_for_lookup(self.client.clone()) .await @@ -1883,6 +1897,9 @@ impl KubernetesComputeDriver { .into_iter() .next() .ok_or(KubernetesDriverError::NotFound)?; + // Proxy-pod companion resources are named from the sandbox name, which + // is not the CR name. + let sandbox_name = annotation_or_label(&object, LABEL_SANDBOX_NAME); let namespace = object .metadata .namespace @@ -1936,6 +1953,7 @@ impl KubernetesComputeDriver { Ok(( agent_sandbox_api, kube_name, + sandbox_name, pod_name, namespace, stop_timeout, @@ -1954,64 +1972,72 @@ impl KubernetesComputeDriver { .await?; let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); - let (kube_name, obj_namespace, _workspace, preconditions) = match tokio::time::timeout( - KUBE_API_TIMEOUT, - lookup_api.api.list(&lp), - ) - .await - { - Ok(Ok(list)) => { - if let Some(obj) = list.items.into_iter().next() { - match obj.metadata.name { - Some(name) => { - let ns = obj - .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - let ws = obj - .metadata - .labels - .as_ref() - .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) - .unwrap_or_default(); - let pc = Preconditions { - uid: obj.metadata.uid, - resource_version: obj.metadata.resource_version, - }; - (name, ns, ws, pc) + let (kube_name, sandbox_name, obj_namespace, _workspace, preconditions) = + match tokio::time::timeout(KUBE_API_TIMEOUT, lookup_api.api.list(&lp)).await { + Ok(Ok(list)) => { + if let Some(obj) = list.items.into_iter().next() { + // Per-sandbox proxy-pod resources are named from the + // sandbox name, not the CR name. They differ: a CR is + // `--`. + let sandbox_name = annotation_or_label(&obj, LABEL_SANDBOX_NAME); + match obj.metadata.name { + Some(name) => { + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let ws = obj + .metadata + .labels + .as_ref() + .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) + .unwrap_or_default(); + let pc = Preconditions { + uid: obj.metadata.uid, + resource_version: obj.metadata.resource_version, + }; + (name, sandbox_name, ns, ws, pc) + } + None => return Ok(false), } - None => return Ok(false), + } else { + debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)"); + return Ok(false); } - } else { - debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)"); - return Ok(false); } - } - Ok(Err(err)) => { - warn!( - sandbox_id = %sandbox_id, - error = %err, - "Failed to list sandbox for deletion from Kubernetes" - ); - return Err(err.to_string()); - } - Err(_elapsed) => { + Ok(Err(err)) => { + warn!( + sandbox_id = %sandbox_id, + error = %err, + "Failed to list sandbox for deletion from Kubernetes" + ); + return Err(err.to_string()); + } + Err(_elapsed) => { + warn!( + sandbox_id = %sandbox_id, + timeout_secs = KUBE_API_TIMEOUT.as_secs(), + "Timed out listing sandbox for deletion from Kubernetes" + ); + return Err(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )); + } + }; + + if self.config.topology == SupervisorTopology::ProxyPod { + if let Some(sandbox_name) = sandbox_name.as_deref() { + self.cleanup_proxy_pod_resources(sandbox_name, &obj_namespace) + .await; + } else { warn!( sandbox_id = %sandbox_id, - timeout_secs = KUBE_API_TIMEOUT.as_secs(), - "Timed out listing sandbox for deletion from Kubernetes" + kube_name = %kube_name, + "Sandbox CR has no sandbox-name label; leaving proxy-pod resources to owner-reference GC" ); - return Err(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() - )); } - }; - - if self.config.topology == SupervisorTopology::ProxyPod { - self.cleanup_proxy_pod_resources(&kube_name, &obj_namespace) - .await; } let delete_api = self @@ -7850,6 +7876,28 @@ mod tests { assert!(err.contains("proxy-pod"), "{err}"); } + /// Per-sandbox proxy-pod resources are named from the sandbox name, not + /// the Sandbox CR name -- the CR is `--`. Deriving them + /// from the CR name silently targets objects that do not exist, which + /// owner-reference GC then masks on delete but not on stop/start. + #[test] + fn proxy_pod_resource_names_come_from_the_sandbox_name_not_the_cr_name() { + let from_sandbox_name = proxy_pod_resource_names("rdy"); + let from_cr_name = proxy_pod_resource_names("default--rdy"); + + assert_ne!( + from_sandbox_name.supervisor_deployment, + from_cr_name.supervisor_deployment + ); + assert!( + from_sandbox_name + .supervisor_deployment + .starts_with("os-sup-rdy-"), + "{}", + from_sandbox_name.supervisor_deployment + ); + } + #[test] fn proxy_pod_reports_no_supervisor_session_model() { let obj = sandbox_object_with_conditions(&[("Ready", "True")]); From dc65ac40f2bd1d67b45df05e36f9ae0edf8856d1 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 14:58:40 -0400 Subject: [PATCH 13/25] docs(rfc): correct the readiness and workload-command analysis The earlier draft framed these as two independent gaps and said the driver silently discarded the workload command. That was wrong about the mechanism: the initial command from 'sandbox create' is delivered over the supervisor session after Ready, so it never ran because Ready never arrived. Rewrite both sections around what the code actually does -- Ready gated on a relay-carrying session that this topology cannot open -- and record the fixes and their cluster verification, including the CR-name versus sandbox-name bug that only on-cluster testing exposed. Signed-off-by: Russell Bryant --- .../openshell-driver-kubernetes/src/driver.rs | 2 +- rfc/proxy-pod-topology-DRAFT.md | 144 +++++++++++------- 2 files changed, 92 insertions(+), 54 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index cbfd4fc75f..057869df00 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1699,7 +1699,7 @@ impl KubernetesComputeDriver { KUBE_API_TIMEOUT, deployments.patch( &names.supervisor_deployment, - &PatchParams::apply("openshell-driver-kubernetes").force(), + &PatchParams::default(), &Patch::Merge(&patch), ), ) diff --git a/rfc/proxy-pod-topology-DRAFT.md b/rfc/proxy-pod-topology-DRAFT.md index 29f58ed6f4..32e0cd48de 100644 --- a/rfc/proxy-pod-topology-DRAFT.md +++ b/rfc/proxy-pod-topology-DRAFT.md @@ -47,9 +47,10 @@ built-in `nonroot-v2` SCC and needs a gated Helm grant, not a custom SCC. Validation confirmed the security model works as designed on OpenShift — unproxied egress denied, proxied egress policy-evaluated, resources -garbage-collected — and surfaced two usability gaps that block adoption: the -user-supplied workload command is silently discarded, and sandboxes never leave -the `Provisioning` phase. +garbage-collected — and surfaced two adoption blockers, both since fixed and +re-verified: sandboxes never left the `Provisioning` phase because readiness +was gated on a supervisor session this topology cannot have, and the workload +container had no way to be given a long-running command. ## Motivation @@ -375,42 +376,68 @@ better operational default for latency-sensitive agents; `required` risks unschedulable pairs under node pressure. The default is left at `disabled` in this RFC but is a reasonable thing for reviewers to push back on. -### Two gaps that block usability - -Cluster validation surfaced two problems that are not OpenShift-specific and -that this RFC treats as required work, not follow-ups. - -**The workload command has nowhere to go.** In `combined` and `sidecar` the -agent container's command is the supervisor binary, and the user's command -reaches the workload through the gateway session. `proxy-pod` has no supervisor -and no session, and `DriverSandboxTemplate` carries no `command`/`args` field at -all, so `openshell sandbox create -- ` is accepted and then silently -discarded. Worse, OpenShell's own sandbox images have `/bin/bash` as their -entrypoint, which under kubelet with no TTY reads EOF and exits 0 immediately — -so the default image produces a `CrashLoopBackOff` with empty logs. Verified: a -`proxy-pod` sandbox on the stock base image crashlooped, and only an image with -a genuinely long-running entrypoint stayed up. - -Options are to add `command`/`args` to `DriverSandboxTemplate` (a proto change -affecting every driver), to accept them through the Kubernetes driver's -`platform_config` passthrough (driver-local, no proto change), or to reject the -combination at the API boundary. At minimum the gateway must not silently -discard a command the user supplied. - -**Sandboxes never reach `Ready`.** The gateway drives the `Ready` transition -from the supervisor session, which the process supervisor in the agent -container opens. `proxy-pod` has no process supervisor, so nothing opens that -session and the sandbox sits in `Provisioning` forever — even though the -Kubernetes `Sandbox` CR reports `Ready`/`DependenciesReady`, both pods are -running, and policy-enforced egress works end to end. Every `Ready`-gated RPC -is then unreachable: `sandbox stop` fails with *"sandbox must be Ready to stop -(current phase: Provisioning)"*, which in turn makes the supervisor scale-down -proposed above unreachable in practice. - -This needs a readiness path that does not assume an in-pod process supervisor — -most naturally the network supervisor reporting readiness for its paired -sandbox once its proxy is serving, since it already holds the gateway -credentials and polls for policy. +### Readiness without a supervisor session + +`SandboxPhase::Ready` was reachable only through a live `ConnectSupervisor` +session. That session is opened solely by `openshell-supervisor-process`, and +its `GatewayMessage` payload is relays — `RelayOpen`/`RelayClose` — plus session +control and heartbeats. So `Ready` has meant "the gateway can open relays into +this sandbox," which for `proxy-pod` will never be true and should not be. + +Left alone, this made the topology unusable: on OpenShift both pods ran and +policy-enforced egress worked end to end while the sandbox reported +`Provisioning` indefinitely, and every `Ready`-gated RPC — including `stop` and +`start` — was unreachable. + +This RFC proposes making the readiness contract explicit rather than implied. A +`SupervisorSessionModel` on `DriverSandboxStatus` lets a driver declare that a +sandbox has no in-sandbox process supervisor. `UNSPECIFIED` preserves the +existing behavior, so drivers that never set it are unaffected; the Kubernetes +driver reports `NONE` for `proxy-pod` and `REQUIRED` otherwise. The gateway then +derives readiness for such sandboxes from the backend conditions alone. + +Two consequences fall out of that and are part of the proposal: + +**Readiness must not become a lie.** With the session gate removed, `Ready` +follows the agent pod, which says nothing about whether the paired supervisor is +serving. A pod could be Ready with no egress path at all. The agent pod +therefore gains a `wait-for-proxy` init container that blocks until the paired +supervisor accepts connections on its proxy port, so pod readiness transitively +means egress works. This also closes a pre-existing ordering gap where the +workload could start before the proxy existed and its early requests simply +failed. + +**Relay-backed RPCs must fail honestly.** Once such sandboxes reach `Ready`, +`exec`, `connect`, port forwarding, and file transfer would pass their readiness +checks and then wait out a session timeout that cannot succeed. The same +declaration lets the gateway reject them immediately with an error naming the +topology. + +### Running a workload with no supervisor to launch it + +`proxy-pod` runs the sandbox image directly. Nothing supplies a command: the +initial command from `openshell sandbox create -- ` is delivered over the +supervisor session as an exec/SSH session after `Ready`, which this topology +does not have, and `DriverSandboxTemplate` has no `command`/`args` field. + +That is tolerable for images built to run a workload, but OpenShell's own +sandbox images use an interactive shell entrypoint. Under kubelet with no TTY it +reads EOF and exits 0, so the stock image produces a `CrashLoopBackOff` with +empty logs — verified on OpenShift, where only an image with a genuinely +long-running entrypoint stayed up. + +This RFC proposes accepting `containers.agent.command` and +`containers.agent.args` through the Kubernetes driver's existing `driver_config` +passthrough, alongside `resources` and `volume_mounts`. That needs no public API +change and reuses the documented escape hatch for driver-specific settings. The +fields are rejected in `combined` and `sidecar`, where the driver replaces the +container command with the supervisor binary and an override would be accepted +and then silently dropped. + +Adding `command`/`args` to the public `SandboxTemplate` remains the more +discoverable long-term answer, but it forces a semantic decision — the field is +genuinely inapplicable to topologies where the supervisor is the entrypoint — and +is deferred rather than resolved here. ### Feature availability @@ -460,11 +487,22 @@ chart, then deployed to OpenShift 4.22.6 / OVN-Kubernetes. Measured results: | Policy-denied host through the proxy | pass, 403 at CONNECT | | Policy-allowed host through the proxy | pass, HTTP 200 with the generated CA trusted | | All resources reclaimed on delete | pass | -| Sandbox reaches `Ready` | **fail** — stuck in `Provisioning` | -| `sandbox stop` / `start` | **blocked** by the `Ready` gate | +| Sandbox reaches `Ready` | pass, after the `SupervisorSessionModel` change | +| `wait-for-proxy` init container gates pod readiness | pass | +| Relay RPCs rejected with a topology error | pass, 43ms rather than a timeout | +| `sandbox stop` scales the supervisor to zero | pass | +| `sandbox start` scales it back and returns to service | pass | +| Stock sandbox image runs via `containers.agent.command` | pass, previously `CrashLoopBackOff` | + +Cluster testing also caught a bug the unit tests could not: the stop, start, +and delete paths derived per-sandbox resource names from the `Sandbox` CR name +rather than the sandbox name, which differ (`default--rdy` versus `rdy`). The +scale-down silently patched a Deployment that does not exist, and delete was +affected too but owner-reference garbage collection reclaimed the resources and +hid it. The remaining work is documenting the OpenShift path in -`docs/kubernetes/openshift.mdx` and closing the two gaps above. +`docs/kubernetes/openshift.mdx`. **Phase 4 — test strategy.** The branch adds `mise run e2e:kubernetes:proxy-pod`, but its `PROXY_POD_E2E` flag currently only prints warnings — it gates nothing. @@ -494,10 +532,8 @@ whose failure mode is invisible. properties may not anticipate that `openshell sandbox exec` and `connect` simply stop working. The gateway should reject those RPCs for `proxy-pod` sandboxes with an actionable error naming the topology, rather than failing obscurely. -The observed behavior today is worse than obscure: a working sandbox reports -`Provisioning` indefinitely and a supplied command is discarded without a -warning, so the failure looks like a broken deployment rather than an -intentional topology limit. +This is now the behavior: relay-backed RPCs are rejected immediately with an +error naming the topology and pointing at `combined` or `sidecar`. **Resource multiplication.** Every sandbox becomes two pods plus three supporting objects. At scale this doubles pod count, doubles scheduling @@ -581,14 +617,16 @@ non-default DNS deployments. Configuration handles every case with no new RBAC. - Should a startup fence-verification probe be a **requirement** for graduating `proxy-pod` out of experimental, given that the failure mode of a non-enforcing CNI is silent? -- Should the network supervisor own the `Ready` transition for its paired - sandbox, or should the gateway derive `Ready` from the `Sandbox` CR conditions - when the topology has no process supervisor? -- Should the workload command reach the container through a new - `DriverSandboxTemplate` field or through the Kubernetes driver's - `platform_config` passthrough? +- Should `command`/`args` graduate from the Kubernetes `driver_config` + passthrough to the public `SandboxTemplate`, and if so what do they mean in + topologies where the supervisor is the container entrypoint? +- Should `openshell sandbox create -- ` be reinterpreted as the container + command in topologies with no session, rather than failing to deliver it? - Should OpenShell publish a `proxy-pod`-suitable sandbox image with a - long-running entrypoint, given that the current images crashloop here? + long-running entrypoint, so the default path works without `driver_config`? +- Should a future `SupervisorSessionModel` variant carry a capability list, so + the gateway can gate individual RPCs rather than treating relays as + all-or-nothing? - Should `affinity` default to `preferred` rather than `disabled`, given that the default sends all workload egress across nodes? - Should the gateway reject `exec`/`connect`/`upload`/`sync` for `proxy-pod` From 0bb909f3e37b3159fd4abff394ed29e73a40b6d0 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 15:21:36 -0400 Subject: [PATCH 14/25] docs(rfc): explain the relay boundary and observability tradeoffs The feature list said SSH, exec, and file transfer were unavailable without saying why, which read as an implementation gap rather than a structural one. Record the mechanism: RelayOpen targets something 'inside the sandbox', the SSH server exists only in openshell-supervisor-process, and sessions need the workload's PID, mount, and network namespaces -- ssh.rs calls setns to enter the sandbox netns. The sidecar bridge to an abstract socket works only because both processes share a pod. Note that TCP relays are the exception and are recoverable for services bound to 0.0.0.0. Add the observability picture, measured on OpenShift. Network OCSF events, policy config events, and denial analysis all reach 'openshell logs' as usual, because log push is gated on sandbox ID and endpoint rather than topology. What is lost is workload stdout, which now reaches only the container log, and actor attribution on network events, which renders as -(0) because reading /proc across a pod boundary is impossible. Restructure the compatibility tables by concern and record the enforcement mechanism, pods per sandbox, and OpenShift SCC per topology. Signed-off-by: Russell Bryant --- rfc/proxy-pod-topology-DRAFT.md | 130 ++++++++++++++++++++++++++++++-- 1 file changed, 122 insertions(+), 8 deletions(-) diff --git a/rfc/proxy-pod-topology-DRAFT.md b/rfc/proxy-pod-topology-DRAFT.md index 32e0cd48de..79f89756a7 100644 --- a/rfc/proxy-pod-topology-DRAFT.md +++ b/rfc/proxy-pod-topology-DRAFT.md @@ -439,24 +439,138 @@ discoverable long-term answer, but it forces a semantic decision — the field i genuinely inapplicable to topologies where the supervisor is the entrypoint — and is deferred rather than resolved here. +### Why relays cannot cross the pod boundary + +The relay protocol states the constraint directly: `RelayOpen`'s target is +"the target the supervisor should dial **inside the sandbox**." Every +relay-backed capability — SSH, `exec`, port forwarding, file transfer — is a +request to reach into the sandbox and connect to something. Three properties +make that impossible from a separate pod: + +- **The SSH server exists only in the process supervisor.** `russh` is a + dependency of `openshell-supervisor-process` and the gateway. + `openshell-supervisor-network`, the only supervisor `proxy-pod` runs, has no + SSH server at all. +- **Sessions must land in the workload's namespaces.** `ssh.rs` spawns PTY + shells and pipe-execs that need the workload's PID, mount, and user + namespaces, and for networking it calls `setns(fd, CLONE_NEWNET)` on a + dedicated thread to enter the sandbox network namespace — otherwise + connections reach the host loopback rather than the sandbox loopback where + services listen. A supervisor in another pod holds none of those namespaces. +- **The `sidecar` bridge does not generalize.** In `sidecar` the network + sidecar owns the gateway session but does not serve SSH itself; it bridges + relays to a Linux abstract socket owned by the process supervisor in the + agent container, verified by peer PID. That works only because both run in + one pod. + +SSHing into the supervisor pod would land a shell in the wrong container. + +One nuance is worth recording, because it narrows the gap. `RelayOpen` also +carries a `TcpRelayTarget`, used for port forwarding and service exposure, and +that is *not* structurally impossible here: the supervisor pod can dial the +agent pod's IP, since this design restricts agent **egress** and supervisor +**ingress** but leaves agent ingress open. The obstacle is practical rather +than architectural — `connect_in_netns` exists precisely because workloads +usually bind `127.0.0.1`, which is unreachable across pods, so it would work +for services bound to `0.0.0.0` and fail otherwise. The current implementation +rejects all relays uniformly, which is correct and safe; restoring TCP relays +alone is possible later and is the strongest argument for giving +`SupervisorSessionModel` a capability list rather than treating relays as +all-or-nothing. + +### Observability + +Network-layer observability survives intact; anything requiring visibility +inside the workload's namespaces does not. Log push to the gateway is gated on +the sandbox ID and gateway endpoint rather than on topology, and the proxy pod +has both, so `openshell logs ` carries `[sandbox]` lines as usual. +Confirmed on OpenShift: + +```text +[sandbox] [OCSF] NET:OPEN [MED] DENIED -(0) -> github.com:443 [engine:opa] [reason:network connections not allowed by policy] +[sandbox] [OCSF] CONFIG:LOADED [INFO] Acknowledged initial policy revision as loaded [version:1] +[sandbox] Flushed denial analysis to gateway proposals=2 summaries=2 +``` + +| Signal | `proxy-pod` | +|---|---| +| `NET:*` allow/deny with policy engine and reason | full | +| `CONFIG:*` policy and inference-route changes | full | +| Activity summaries and denial analysis for the policy advisor | full | +| Gateway-side logs | full | +| Workload stdout/stderr | **container log only** (`kubectl logs`), never `openshell logs` | +| Process and binary attribution on network events | **none** | +| `PROCESS:*`, `SSH:*`, Landlock/filesystem events | **none** | + +Two losses deserve emphasis. The workload's own output is no longer captured +by OpenShell at all: the workload is the container's PID 1 and no OpenShell +process shares that pod, so its output reaches only the container log. Anyone +driving OpenShell through the API rather than with cluster access cannot see it. + +And network events carry no actor: the denial above reads `-(0)`, an empty +process name and PID 0. Binary-aware attribution requires reading +`/proc/` across the workload's PID namespace, which a separate pod cannot +do. Operators can therefore answer what was denied but not which process +attempted it, which removes `policy.binaries` as both an enforcement and a +forensic tool. + ### Feature availability +#### Enforcement + | Capability | `combined` | `sidecar` | `cni-sidecar` | `proxy-pod` | |---|---|---|---|---| | Network endpoint + L7 policy | yes | yes | yes | yes | +| Enforcement mechanism | in-pod nftables | in-pod nftables | node CNI rules | **`NetworkPolicy`** | | Filesystem policy | yes | partial (Landlock) | partial (Landlock) | **no** | | Process / binary identity | yes | yes | yes | **no** | -| SSH / `connect` | yes | yes | yes | **no** | -| `exec` | yes | yes | yes | **no** | -| Upload / download / sync | yes | yes | yes | **no** | +| `policy.binaries` matching | yes | yes | yes | **no** — no actor attribution | | Dynamic provider env injection | yes | yes | yes | **no** | + +#### Session and file access + +All relay-backed, and all requiring the workload's namespaces: + +| Capability | `combined` | `sidecar` | `cni-sidecar` | `proxy-pod` | +|---|---|---|---|---| +| SSH / `connect` | yes | yes | yes | **no** — structurally impossible | +| `exec` | yes | yes | yes | **no** — structurally impossible | +| Upload / download / sync | yes | yes | yes | **no** — structurally impossible | +| Port forwarding / service exposure | yes | yes | yes | **no today** — recoverable for `0.0.0.0` binds | +| Initial command from `sandbox create -- ` | yes | yes | yes | **no** — use `containers.agent.command` | + +#### Observability + +| Signal | `combined` | `sidecar` | `cni-sidecar` | `proxy-pod` | +|---|---|---|---|---| +| `NET:*` allow/deny with reason | yes | yes | yes | yes | +| `CONFIG:*` policy and route changes | yes | yes | yes | yes | +| Denial analysis for the policy advisor | yes | yes | yes | yes | +| Workload stdout/stderr in `openshell logs` | yes | yes | yes | **no** — container log only | +| Actor process on network events | yes | yes | yes | **no** — renders as `-(0)` | +| `PROCESS:*` lifecycle events | yes | yes | yes | **no** | +| `SSH:*` events | yes | yes | yes | **no** | +| Landlock / filesystem events | yes | partial | partial | **no** | + +#### Operational posture + +| Property | `combined` | `sidecar` | `cni-sidecar` | `proxy-pod` | +|---|---|---|---|---| | Privileged init container | no | **yes** | no | no | | Added capabilities in sandbox pod | **yes** | no | no | no | -| Requires NetworkPolicy enforcement | no | no | no | **yes** | - -The sandbox image's own entrypoint and command determine what runs. This -topology suits batch and autonomous agent workloads that need policy-enforced -egress and never need an interactive session. +| Node-level privileged DaemonSet | no | no | **yes** | no | +| Requires `NetworkPolicy` enforcement | no | no | no | **yes** | +| Pods per sandbox | 1 | 1 | 1 | **2** | +| OpenShift SCC required | `privileged` | custom | custom + `privileged` CNI | **built-in `nonroot-v2`** | + +The dividing line is consistent: everything observable or enforceable at the +network boundary survives, and everything needing visibility inside the +workload's namespaces does not. `proxy-pod` suits batch and autonomous agent +workloads that need policy-enforced egress, ship their own long-running +entrypoint, and never need a human on the other end. Operators who want the +interactive workflow *and* low pod privilege should use `cni-sidecar`, which +keeps the full supervisor contract at the cost of a custom SCC and a +node-level DaemonSet. The two are complementary, not competing. ## Implementation plan From 9e0647e9f66e2eb281015146a5e07f72618da030 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 15:35:10 -0400 Subject: [PATCH 15/25] fix(cli): explain sessionless topologies instead of surfacing a raw error Creating a sandbox in a topology with no in-sandbox supervisor succeeded, then failed at the interactive-session step with a bare gRPC error. The sandbox was running and its network policy enforced, but the output read as a failed create. Detect the gateway's rejection and print what actually happened: the sandbox is running, sessions are unavailable for this topology, egress is unaffected, and which topologies to use when interactive access is required. When a command was passed to 'sandbox create', say plainly that it did not run and point at the containers.agent.command entrypoint override instead. The command still exits non-zero. A command that did not run must not report success, and callers should not have to parse output to find that out. Detection keys on a stable marker constant shared through openshell-core rather than on prose, so rewording the message cannot silently break it, and it searches the whole error chain because the marker arrives wrapped in a transport error. Signed-off-by: Russell Bryant --- crates/openshell-cli/src/run.rs | 105 ++++++++++++++++++ crates/openshell-core/src/error.rs | 21 ++++ .../src/supervisor_session.rs | 4 +- 3 files changed, 127 insertions(+), 3 deletions(-) diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 5147ee2831..538b0e8bb3 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -333,6 +333,55 @@ fn validate_memory_quantity(value: &str) -> Result { Ok(value.to_string()) } +/// True when an error is the gateway rejecting a relay-backed operation because +/// the sandbox's topology has no in-sandbox supervisor. +fn is_no_supervisor_session_error(err: &miette::Report) -> bool { + let marker = openshell_core::error::NO_SUPERVISOR_SESSION_MARKER; + // Check the whole chain: the marker may sit in a wrapped transport error + // rather than the outermost message. + format!("{err}").contains(marker) + || err + .chain() + .any(|source| source.to_string().contains(marker)) +} + +/// Explain a topology that cannot open sessions, instead of letting a raw gRPC +/// error imply the sandbox failed to start. +/// +/// The sandbox is running and its network policy is enforced; only the +/// interactive path is unavailable. The command still exits non-zero, because +/// a command passed to `sandbox create` did not run and callers must not read +/// success from the exit code. +fn report_no_supervisor_session(sandbox_name: &str, had_command: bool, persisted: bool) { + eprintln!(); + eprintln!( + "{} Sandbox '{}' is running, but this topology cannot open sessions.", + "!".yellow().bold(), + sandbox_name.bold() + ); + eprintln!(" SSH, exec, port forwarding, and file transfer need a supervisor inside"); + eprintln!(" the sandbox, which this topology does not run."); + eprintln!(); + if had_command { + eprintln!(" {} your command did not run.", "Note:".bold()); + eprintln!(" Set the workload entrypoint instead, so it starts with the container:"); + eprintln!( + " --driver-config-json '{{\"kubernetes\":{{\"containers\":{{\"agent\":{{\"command\":[...]}}}}}}}}'" + ); + } else { + eprintln!(" Policy-enforced network egress is unaffected."); + } + eprintln!(); + if persisted { + eprintln!(" Inspect it with:"); + eprintln!(" openshell logs {sandbox_name}"); + eprintln!(" openshell sandbox list"); + } + eprintln!(" Use the `combined`, `sidecar`, or `cni-sidecar` topology when you need"); + eprintln!(" interactive sessions."); +} + +#[allow(clippy::too_many_arguments)] async fn finalize_sandbox_create_session( server: &str, sandbox_name: &str, @@ -341,8 +390,20 @@ async fn finalize_sandbox_create_session( workspace: &str, tls: &TlsOptions, gateway: &str, + had_command: bool, ) -> Result<()> { + let sessionless = session_result + .as_ref() + .err() + .is_some_and(is_no_supervisor_session_error); + if persist { + if sessionless { + report_no_supervisor_session(sandbox_name, had_command, true); + return Err(miette::miette!( + "sandbox '{sandbox_name}' is running but cannot open interactive sessions" + )); + } return session_result; } @@ -354,6 +415,15 @@ async fn finalize_sandbox_create_session( eprintln!("Failed to delete sandbox {sandbox_name}: {err}"); } + if sessionless { + // The sandbox has already been deleted per --no-keep, so do not point + // the reader at commands that would now fail. + report_no_supervisor_session(sandbox_name, had_command, false); + return Err(miette::miette!( + "sandbox '{sandbox_name}' could not open an interactive session" + )); + } + session_result } @@ -987,6 +1057,7 @@ pub async fn sandbox_create( workspace, &effective_tls, gateway_name, + !command.is_empty(), ) .await } @@ -1011,6 +1082,7 @@ pub async fn sandbox_create( workspace, &effective_tls, gateway_name, + !command.is_empty(), ) .await } @@ -7704,6 +7776,39 @@ mod tests { assert!(sandbox_should_persist(true, None)); } + use crate::run::is_no_supervisor_session_error; + + #[test] + fn detects_the_no_supervisor_session_rejection() { + let err = miette::miette!("{}", openshell_core::error::no_supervisor_session_message()); + assert!(is_no_supervisor_session_error(&err)); + } + + #[test] + fn other_errors_are_not_mistaken_for_a_sessionless_topology() { + for message in [ + "supervisor session not connected", + "sandbox not found", + "timed out waiting for the sandbox to become ready", + ] { + let err = miette::miette!("{message}"); + assert!( + !is_no_supervisor_session_error(&err), + "{message} must not be treated as a sessionless topology" + ); + } + } + + /// The marker travels through gRPC as part of the status message, so + /// detection has to survive the wrapping the transport and CLI add. + #[test] + fn detection_survives_error_wrapping() { + let inner = + Status::failed_precondition(openshell_core::error::no_supervisor_session_message()); + let err = miette::miette!("failed to open session: {inner}"); + assert!(is_no_supervisor_session_error(&err)); + } + #[test] fn sandbox_should_not_persist_when_no_keep_is_set() { assert!(!sandbox_should_persist(false, None)); diff --git a/crates/openshell-core/src/error.rs b/crates/openshell-core/src/error.rs index 8c23e30198..16032c7d4e 100644 --- a/crates/openshell-core/src/error.rs +++ b/crates/openshell-core/src/error.rs @@ -138,3 +138,24 @@ impl From for tonic::Status { } } } + +/// Stable marker embedded in the gateway's rejection of relay-backed RPCs for +/// sandboxes whose topology has no in-sandbox process supervisor. +/// +/// SSH, `exec`, port forwarding, and file transfer all travel over the +/// supervisor session, which such a topology never opens. The CLI matches on +/// this marker to explain the situation rather than surfacing a raw gRPC +/// error, so callers must keep the two in sync. It is deliberately a distinct +/// token rather than prose so rewording the message cannot break detection. +pub const NO_SUPERVISOR_SESSION_MARKER: &str = "openshell:no-supervisor-session"; + +/// Full message returned for relay-backed RPCs against such a sandbox. +#[must_use] +pub fn no_supervisor_session_message() -> String { + format!( + "this sandbox's topology runs no supervisor inside the sandbox, so SSH, exec, port \ + forwarding, and file transfer are unavailable. Policy-enforced network egress is \ + unaffected. Use the `combined`, `sidecar`, or `cni-sidecar` topology when interactive \ + sessions are required. [{NO_SUPERVISOR_SESSION_MARKER}]" + ) +} diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index 9fe51c2860..6e271cf679 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -191,9 +191,7 @@ impl SupervisorSessionRegistry { // burning the caller's timeout on a wait that cannot succeed. if self.is_sessionless(sandbox_id) { return Err(Status::failed_precondition( - "this sandbox runs a topology with no in-sandbox supervisor, so SSH, exec, \ - port forwarding, and file transfer are unavailable; use the `combined` or \ - `sidecar` topology when those are required", + openshell_core::error::no_supervisor_session_message(), )); } From 2a162788a7ce317a14d912ae4ccd63d524a0c15d Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 15:57:10 -0400 Subject: [PATCH 16/25] fix(cli): detect a sessionless topology before opening a session The previous commit explained the topology after a relay RPC was rejected, but 'sandbox create' still spawned ssh first, so the failure surfaced through the subprocess as 'ssh exited with status 255' and the explanatory message was buried in wrapped stderr. Publish a SupervisorSession=False/NotApplicable condition in the public sandbox status when the driver reports SupervisorSessionModel::None, and have the CLI check it before attempting a session. When set, the CLI skips the connect/exec path entirely and prints the explanation directly: the sandbox is running, sessions are unavailable for this topology, egress is unaffected, and how to set a workload entrypoint when a command was supplied. The error-marker detection from the prior commit stays as the backstop for relay RPCs issued directly against an existing sandbox, where there is no create-time status to pre-check. Signed-off-by: Russell Bryant --- crates/openshell-cli/src/run.rs | 83 +++++++++++++++++++++- crates/openshell-server/src/compute/mod.rs | 51 +++++++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 538b0e8bb3..135aaaa9e2 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -333,6 +333,22 @@ fn validate_memory_quantity(value: &str) -> Result { Ok(value.to_string()) } +/// True when the gateway reports that this sandbox's topology never opens a +/// supervisor session, so relay-backed operations cannot work. +/// +/// Checked before attempting a session rather than after: the failure would +/// otherwise surface through the `ssh` subprocess as `exit status 255`, which +/// tells the reader nothing. +fn sandbox_has_no_supervisor_session(sandbox: &Sandbox) -> bool { + sandbox.status.as_ref().is_some_and(|status| { + status.conditions.iter().any(|condition| { + condition.r#type == "SupervisorSession" + && condition.status.eq_ignore_ascii_case("false") + && condition.reason == "NotApplicable" + }) + }) +} + /// True when an error is the gateway rejecting a relay-backed operation because /// the sandbox's topology has no in-sandbox supervisor. fn is_no_supervisor_session_error(err: &miette::Report) -> bool { @@ -1037,6 +1053,32 @@ pub async fn sandbox_create( return Ok(()); } + // Skip the session entirely when the topology cannot serve one. + // Attempting it would spawn ssh, fail inside the subprocess, and + // surface as an opaque exit status. + if sandbox_has_no_supervisor_session(&last_sandbox) { + let had_command = !command.is_empty(); + if !persist { + let names = [sandbox_name.clone()]; + if let Err(err) = sandbox_delete( + &effective_server, + &names, + false, + workspace, + &effective_tls, + gateway_name, + ) + .await + { + eprintln!("Failed to delete sandbox {sandbox_name}: {err}"); + } + } + report_no_supervisor_session(&sandbox_name, had_command, persist); + return Err(miette::miette!( + "sandbox '{sandbox_name}' cannot open interactive sessions" + )); + } + let connect_result = if persist { sandbox_connect(&effective_server, &sandbox_name, &effective_tls, workspace).await } else { @@ -7776,7 +7818,46 @@ mod tests { assert!(sandbox_should_persist(true, None)); } - use crate::run::is_no_supervisor_session_error; + use crate::run::{is_no_supervisor_session_error, sandbox_has_no_supervisor_session}; + + #[test] + fn detects_the_sessionless_condition_on_a_sandbox() { + use openshell_core::proto::{Sandbox, SandboxCondition, SandboxStatus}; + + let sessionless = Sandbox { + status: Some(SandboxStatus { + conditions: vec![SandboxCondition { + r#type: "SupervisorSession".to_string(), + status: "False".to_string(), + reason: "NotApplicable".to_string(), + message: openshell_core::error::no_supervisor_session_message(), + last_transition_time: String::new(), + }], + ..Default::default() + }), + ..Default::default() + }; + assert!(sandbox_has_no_supervisor_session(&sessionless)); + + // A supervisor that is merely not connected yet must not be mistaken + // for a topology that will never have one. + let still_settling = Sandbox { + status: Some(SandboxStatus { + conditions: vec![SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "SupervisorNotConnected".to_string(), + message: "Backend ready; waiting for supervisor session".to_string(), + last_transition_time: String::new(), + }], + ..Default::default() + }), + ..Default::default() + }; + assert!(!sandbox_has_no_supervisor_session(&still_settling)); + + assert!(!sandbox_has_no_supervisor_session(&Sandbox::default())); + } #[test] fn detects_the_no_supervisor_session_rejection() { diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 7598a3e3d9..a09e4f021b 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -3898,6 +3898,7 @@ struct ComposedPhase { phase: SandboxPhase, session_connected: bool, backend_ready_without_session: bool, + sessionless: bool, } impl ComposedPhase { @@ -3923,6 +3924,7 @@ impl ComposedPhase { backend_ready_without_session: backend_phase == SandboxPhase::Ready && !session_connected && !sessionless, + sessionless, } } @@ -3933,6 +3935,9 @@ impl ComposedPhase { spec: Option<&SandboxSpec>, ) { rewrite_user_facing_conditions(status, spec); + if self.sessionless { + ensure_no_supervisor_session_status(status, sandbox_name); + } if self.backend_ready_without_session { ensure_supervisor_not_connected_status(status, sandbox_name); } else if self.session_connected && self.phase == SandboxPhase::Ready { @@ -3969,6 +3974,52 @@ fn ensure_supervisor_not_ready_status(status: &mut Option, sandbo ); } +/// Condition type advertising whether a sandbox can serve relay-backed +/// operations. +/// +/// Carried in the public status so clients can tell that SSH, `exec`, port +/// forwarding, and file transfer are unavailable *before* attempting one, +/// rather than discovering it from a failed connection. Using a condition +/// avoids adding a field to the public `Sandbox` message. +pub const SUPERVISOR_SESSION_CONDITION: &str = "SupervisorSession"; + +fn upsert_condition( + status: &mut Option, + sandbox_name: &str, + condition: SandboxCondition, +) { + let status = status.get_or_insert_with(|| SandboxStatus { + sandbox_name: sandbox_name.to_string(), + ..Default::default() + }); + + let condition_type = condition.r#type.clone(); + if let Some(existing) = status + .conditions + .iter_mut() + .find(|existing| existing.r#type == condition_type) + { + *existing = condition; + } else { + status.conditions.push(condition); + } +} + +/// Record that this sandbox's topology never opens a supervisor session. +fn ensure_no_supervisor_session_status(status: &mut Option, sandbox_name: &str) { + upsert_condition( + status, + sandbox_name, + SandboxCondition { + r#type: SUPERVISOR_SESSION_CONDITION.to_string(), + status: "False".to_string(), + reason: "NotApplicable".to_string(), + message: openshell_core::error::no_supervisor_session_message(), + last_transition_time: String::new(), + }, + ); +} + fn upsert_ready_condition( status: &mut Option, sandbox_name: &str, From be8ca576b2c2ff6b6ffa9c6579a80a488fec4810 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 17:56:51 -0400 Subject: [PATCH 17/25] docs(rfc): note Kata kernel isolation and clarify workload log location Add the workload-to-supervisor kernel-isolation property to the topology comparison: because proxy-pod places the workload and supervisor in separate pods, a VM-based RuntimeClass like Kata gives them separate VMs and kernels, so a workload kernel compromise does not by itself reach the supervisor's gateway credentials. This is unique to proxy-pod; the in-pod topologies share one Kata VM between workload and supervisor. Clarify that lost workload stdout/stderr is specifically the agent container's log, reachable only via 'kubectl logs ', not 'openshell logs'. Signed-off-by: Russell Bryant --- rfc/proxy-pod-topology-DRAFT.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/rfc/proxy-pod-topology-DRAFT.md b/rfc/proxy-pod-topology-DRAFT.md index 79f89756a7..b25e6ae840 100644 --- a/rfc/proxy-pod-topology-DRAFT.md +++ b/rfc/proxy-pod-topology-DRAFT.md @@ -224,6 +224,17 @@ container performing network supervision. Mounting it into the workload pod would defeat the purpose. This RFC proposes rejecting that combination at configuration validation rather than silently mounting it in the wrong place. +Separate pods also raise the isolation ceiling under a VM-based `RuntimeClass`. +Kata Containers gives each *pod* its own lightweight VM and kernel; containers +within a pod share that VM. In every in-pod topology the workload and the +supervisor live in one pod, so a Kata VM escape — a kernel compromise inside +that shared VM — reaches the supervisor and its gateway credentials. Under +`proxy-pod` the workload and supervisor are separate pods and therefore separate +Kata VMs with separate kernels, so a kernel compromise in the workload VM does +not by itself reach the supervisor. This is unique to `proxy-pod`: it is the +only topology where the workload-to-supervisor boundary can be a hypervisor +boundary rather than a namespace boundary. + ### The NetworkPolicy contract Two policies define the fence: @@ -546,7 +557,7 @@ All relay-backed, and all requiring the workload's namespaces: | `NET:*` allow/deny with reason | yes | yes | yes | yes | | `CONFIG:*` policy and route changes | yes | yes | yes | yes | | Denial analysis for the policy advisor | yes | yes | yes | yes | -| Workload stdout/stderr in `openshell logs` | yes | yes | yes | **no** — container log only | +| Workload stdout/stderr in `openshell logs` | yes | yes | yes | **no** — only in the `agent` container log via `kubectl logs ` | | Actor process on network events | yes | yes | yes | **no** — renders as `-(0)` | | `PROCESS:*` lifecycle events | yes | yes | yes | **no** | | `SSH:*` events | yes | yes | yes | **no** | @@ -561,6 +572,7 @@ All relay-backed, and all requiring the workload's namespaces: | Node-level privileged DaemonSet | no | no | **yes** | no | | Requires `NetworkPolicy` enforcement | no | no | no | **yes** | | Pods per sandbox | 1 | 1 | 1 | **2** | +| Workload/supervisor kernel isolation under Kata | no — one pod, one VM/kernel | no — one pod, one VM/kernel | no — one pod, one VM/kernel | **yes — separate pods, separate Kata VMs/kernels** | | OpenShift SCC required | `privileged` | custom | custom + `privileged` CNI | **built-in `nonroot-v2`** | The dividing line is consistent: everything observable or enforceable at the From 575a447fd8d1be5d3cfd55250906e7d16030926c Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 18:21:54 -0400 Subject: [PATCH 18/25] fix(kubernetes): strip MAIN_PROCESS_SPEC from proxy-pod workloads Main renamed the canonical-command transport from OPENSHELL_SANDBOX_COMMAND to the versioned OPENSHELL_MAIN_PROCESS_SPEC (#2726), which the supervisor decodes and launches. proxy-pod runs the sandbox image directly with no supervisor, so that env var is not only useless in the workload container but leaks the intended command into it. Strip MAIN_PROCESS_SPEC alongside the other supervisor-oriented variables, replacing the now-removed SANDBOX_COMMAND entry. Rebase adaptation: proxy-pod's workload command continues to flow through the containers.agent.command driver_config, since the canonical main process requires an in-sandbox supervisor this topology does not run. Signed-off-by: Russell Bryant --- crates/openshell-core/src/sandbox_env.rs | 3 --- crates/openshell-driver-kubernetes/src/driver.rs | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index c512158334..0e8755e6f9 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -26,9 +26,6 @@ pub const SSH_SOCKET_PATH: &str = "OPENSHELL_SSH_SOCKET_PATH"; /// Log level for the sandbox supervisor (e.g. `"debug"`, `"info"`, `"warn"`). pub const LOG_LEVEL: &str = "OPENSHELL_LOG_LEVEL"; -/// Shell command to run inside the sandbox. -pub const SANDBOX_COMMAND: &str = "OPENSHELL_SANDBOX_COMMAND"; - /// Versioned specification for the exact canonical main process. /// /// Most drivers use JSON directly. Transports that cannot preserve spaces in diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 057869df00..158a22ac6e 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -3741,7 +3741,7 @@ fn apply_supervisor_proxy_pod_topology( openshell_core::sandbox_env::SANDBOX_ID, openshell_core::sandbox_env::SANDBOX, openshell_core::sandbox_env::ENDPOINT, - openshell_core::sandbox_env::SANDBOX_COMMAND, + openshell_core::sandbox_env::MAIN_PROCESS_SPEC, openshell_core::sandbox_env::TELEMETRY_ENABLED, openshell_core::sandbox_env::SSH_SOCKET_PATH, openshell_core::sandbox_env::TLS_CA, From 50f0793df85b4c577f33e12695435ab2e5270eb0 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 18:39:38 -0400 Subject: [PATCH 19/25] fix(cli): warn before implicit detach when a sessionless topology drops the command A non-interactive persistent create takes main's implicit-detach path and returns before the sessionless check ran, so 'openshell sandbox create -- cmd' against a proxy-pod sandbox silently created a sandbox where the command never runs -- exactly the broken-looking outcome the sessionless messaging exists to prevent. Check for a command-bearing sessionless topology before the detach return and explain that the command will not run, pointing at containers.agent.command. The interactive no-command case still reports after detach. Both paths share a new abort_sessionless_create helper so ephemeral cleanup and messaging stay identical. Signed-off-by: Russell Bryant --- crates/openshell-cli/src/run.rs | 80 +++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 24 deletions(-) diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 135aaaa9e2..3e56571075 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -397,6 +397,29 @@ fn report_no_supervisor_session(sandbox_name: &str, had_command: bool, persisted eprintln!(" interactive sessions."); } +/// Delete an ephemeral sandbox, explain the sessionless topology, and return +/// the error to surface. Shared by the pre-detach (command-bearing) and +/// post-detach (interactive) paths so both behave identically. +#[allow(clippy::too_many_arguments)] +async fn abort_sessionless_create( + server: &str, + sandbox_name: &str, + persist: bool, + workspace: &str, + tls: &TlsOptions, + gateway: &str, + had_command: bool, +) -> miette::Report { + if !persist { + let names = [sandbox_name.to_string()]; + if let Err(err) = sandbox_delete(server, &names, false, workspace, tls, gateway).await { + eprintln!("Failed to delete sandbox {sandbox_name}: {err}"); + } + } + report_no_supervisor_session(sandbox_name, had_command, persist); + miette::miette!("sandbox '{sandbox_name}' cannot open interactive sessions") +} + #[allow(clippy::too_many_arguments)] async fn finalize_sandbox_create_session( server: &str, @@ -1042,6 +1065,25 @@ pub async fn sandbox_create( return Ok(()); } + let sessionless = sandbox_has_no_supervisor_session(&last_sandbox); + + // A command given to a sessionless topology never runs: there is no + // supervisor to launch it and no session to exec it. Surface that + // even in the implicit-detach path a non-interactive persistent + // create would otherwise take silently. + if sessionless && !command.is_empty() { + return Err(abort_sessionless_create( + &effective_server, + &sandbox_name, + persist, + workspace, + &effective_tls, + gateway_name, + true, + ) + .await); + } + // Persistent non-interactive creates detach implicitly. An // explicitly ephemeral (`--no-keep`) create must still attach so // it can observe the canonical process and delete the sandbox when @@ -1053,30 +1095,20 @@ pub async fn sandbox_create( return Ok(()); } - // Skip the session entirely when the topology cannot serve one. - // Attempting it would spawn ssh, fail inside the subprocess, and - // surface as an opaque exit status. - if sandbox_has_no_supervisor_session(&last_sandbox) { - let had_command = !command.is_empty(); - if !persist { - let names = [sandbox_name.clone()]; - if let Err(err) = sandbox_delete( - &effective_server, - &names, - false, - workspace, - &effective_tls, - gateway_name, - ) - .await - { - eprintln!("Failed to delete sandbox {sandbox_name}: {err}"); - } - } - report_no_supervisor_session(&sandbox_name, had_command, persist); - return Err(miette::miette!( - "sandbox '{sandbox_name}' cannot open interactive sessions" - )); + // An interactive create against a sessionless topology cannot + // attach. Skip the session — which would spawn ssh, fail inside the + // subprocess, and surface as an opaque exit status — and explain. + if sessionless { + return Err(abort_sessionless_create( + &effective_server, + &sandbox_name, + persist, + workspace, + &effective_tls, + gateway_name, + false, + ) + .await); } let connect_result = if persist { From 3e957f55d2ace6f6ec58e782eb4647d1f8bf37ff Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 20:58:09 -0400 Subject: [PATCH 20/25] fix(kubernetes): correct proxy-pod companion lifecycle, isolation, and placement Addresses several proxy-pod review findings in the Kubernetes driver: - Delete the Sandbox CR (tearing down the workload) before removing the companion resources, so the agent egress NetworkPolicy fence is never dropped while the workload can still egress, and a failed CR delete leaves the fence in place. - Derive companion resource names from the Sandbox CR name, which is unique in every workspace mode, instead of the bare sandbox name. In shared mode workspace-a/dev and workspace-b/dev share a sandbox name, so the previous scheme collided and a rollback could dismantle another sandbox's isolation. - Create companions in, and point the workload's proxy URL at, the sandbox's resolved target namespace rather than the static configured namespace, so proxy-pod works in managed and operator workspace modes. Add the proxy-pod resources to the cluster-scoped Role for those modes. - Remove the raw gateway-forward tunnel (supervisor :18080 to the gateway, reachable by the agent). Nothing on the agent consumed it, and with unauthenticated gateway access it was a policy-bypassing path to the admin API. The supervisor still connects to the gateway directly for its own policy, inference, and log traffic. - Return an error from supervisor Deployment scaling and propagate it from start_sandbox, so a transient scale-up failure surfaces instead of wedging the sandbox in Starting with the supervisor at zero replicas. Scale-down on stop stays best-effort. - Give the supervisor Deployment the workload's nodeSelector, tolerations, and priorityClassName, so required same-node affinity cannot pin the workload to a node its own placement excludes. Signed-off-by: Russell Bryant --- crates/openshell-core/src/sandbox_env.rs | 3 - .../openshell-driver-kubernetes/src/driver.rs | 357 ++++++++++-------- crates/openshell-sandbox/src/lib.rs | 110 ------ .../helm/openshell/templates/clusterrole.yaml | 47 +++ 4 files changed, 247 insertions(+), 270 deletions(-) diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 0e8755e6f9..3fb494e10d 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -143,9 +143,6 @@ pub const NETWORK_BINARY_IDENTITY: &str = "OPENSHELL_NETWORK_BINARY_IDENTITY"; /// container. pub const SIDECAR_CONTROL_SOCKET: &str = "OPENSHELL_SIDECAR_CONTROL_SOCKET"; -/// Address where an external network supervisor forwards gateway gRPC traffic. -pub const GATEWAY_FORWARD_ADDR: &str = "OPENSHELL_GATEWAY_FORWARD_ADDR"; - /// Optional TLS server name override used when connecting to the gateway. pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME"; diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 158a22ac6e..6a299f9054 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1447,7 +1447,7 @@ impl KubernetesComputeDriver { proxy_connect_by_hostname: self.config.proxy_connect_by_hostname == Some(true), proxy_pod_affinity: self.config.proxy_pod.affinity, proxy_pod_dns_peers: &self.config.proxy_pod.dns_peers, - namespace: &self.config.namespace, + namespace: &target_namespace, service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, sandbox_name: &sandbox.name, @@ -1485,7 +1485,9 @@ impl KubernetesComputeDriver { } obj.metadata = ObjectMeta { name: Some(kube_name), - namespace: Some(target_namespace), + // Clone: `params` borrows `target_namespace` for later companion + // creation. + namespace: Some(target_namespace.clone()), labels: Some(sandbox_labels(sandbox, Some(&self.config.gateway_id))), annotations: Some(annotations), ..Default::default() @@ -1567,7 +1569,16 @@ impl KubernetesComputeDriver { sandbox_cr: &DynamicObject, sandbox_api_version: &str, ) -> Result<(), KubernetesDriverError> { - let names = proxy_pod_resource_names(&sandbox.name); + // Companion names derive from the Sandbox CR name, which is unique per + // sandbox in every workspace mode. The bare sandbox name collides in + // shared mode, where `workspace-a/dev` and `workspace-b/dev` both have + // sandbox name `dev`. + let cr_name = sandbox_cr + .metadata + .name + .as_deref() + .unwrap_or(sandbox.name.as_str()); + let names = proxy_pod_resource_names(cr_name); let template_environment = spec .and_then(|spec| spec.template.as_ref()) .map(|template| template.environment.clone()) @@ -1591,20 +1602,26 @@ impl KubernetesComputeDriver { proxy_pod_agent_egress_network_policy(&names, params, dependent_owner_ref.clone()); let supervisor_ingress = proxy_pod_supervisor_ingress_network_policy(&names, params, dependent_owner_ref); + // Give the supervisor the workload's node placement so same-node + // affinity resolves to a node the workload can also use. + let pod_driver_config = spec + .and_then(|spec| spec.template.as_ref()) + .and_then(|template| KubernetesSandboxDriverConfig::from_template(template).ok()) + .map(|config| config.pod) + .unwrap_or_default(); let supervisor_deployment = proxy_pod_supervisor_deployment( &names, &template_environment, &spec_environment, params, + &pod_driver_config, deployment_owner_ref, ); - let secrets: Api = Api::namespaced(self.client.clone(), &self.config.namespace); - let services: Api = Api::namespaced(self.client.clone(), &self.config.namespace); - let policies: Api = - Api::namespaced(self.client.clone(), &self.config.namespace); - let deployments: Api = - Api::namespaced(self.client.clone(), &self.config.namespace); + let secrets: Api = Api::namespaced(self.client.clone(), params.namespace); + let services: Api = Api::namespaced(self.client.clone(), params.namespace); + let policies: Api = Api::namespaced(self.client.clone(), params.namespace); + let deployments: Api = Api::namespaced(self.client.clone(), params.namespace); tokio::time::timeout( KUBE_API_TIMEOUT, @@ -1688,14 +1705,24 @@ impl KubernetesComputeDriver { /// scale up is retried by the agent pod's connection attempts and surfaces /// as a normal readiness failure. Neither should fail the caller's /// start/stop RPC. - async fn scale_proxy_pod_supervisor(&self, sandbox_name: &str, namespace: &str, replicas: u32) { + /// Scale a proxy-pod sandbox's supervisor Deployment. + /// + /// `cr_name` is the Sandbox CR resource name, which is unique per sandbox in + /// every workspace mode (shared prefixes it with the workspace); the bare + /// sandbox name is not. `namespace` is the sandbox's resolved namespace. + async fn scale_proxy_pod_supervisor( + &self, + cr_name: &str, + namespace: &str, + replicas: u32, + ) -> Result<(), KubernetesDriverError> { if self.config.topology != SupervisorTopology::ProxyPod { - return; + return Ok(()); } - let names = proxy_pod_resource_names(sandbox_name); + let names = proxy_pod_resource_names(cr_name); let deployments: Api = Api::namespaced(self.client.clone(), namespace); let patch = serde_json::json!({"spec": {"replicas": replicas}}); - let result = tokio::time::timeout( + match tokio::time::timeout( KUBE_API_TIMEOUT, deployments.patch( &names.supervisor_deployment, @@ -1703,28 +1730,23 @@ impl KubernetesComputeDriver { &Patch::Merge(&patch), ), ) - .await; - match result { - Ok(Ok(_)) => info!( - sandbox_name = %sandbox_name, - deployment = %names.supervisor_deployment, - replicas, - "Scaled proxy-pod supervisor Deployment" - ), - Ok(Err(err)) => warn!( - sandbox_name = %sandbox_name, - deployment = %names.supervisor_deployment, - replicas, - error = %err, - "Failed to scale proxy-pod supervisor Deployment" - ), - Err(_elapsed) => warn!( - sandbox_name = %sandbox_name, - deployment = %names.supervisor_deployment, - replicas, - timeout_secs = KUBE_API_TIMEOUT.as_secs(), - "Timed out scaling proxy-pod supervisor Deployment" - ), + .await + { + Ok(Ok(_)) => { + info!( + cr_name = %cr_name, + deployment = %names.supervisor_deployment, + replicas, + "Scaled proxy-pod supervisor Deployment" + ); + Ok(()) + } + Ok(Err(err)) => Err(KubernetesDriverError::from_kube(err)), + Err(_elapsed) => Err(KubernetesDriverError::Message(format!( + "timed out after {}s scaling proxy-pod supervisor Deployment {}", + KUBE_API_TIMEOUT.as_secs(), + names.supervisor_deployment + ))), } } @@ -1766,7 +1788,7 @@ impl KubernetesComputeDriver { } pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let (agent_sandbox_api, kube_name, sandbox_name, pod_name, namespace, stop_timeout) = self + let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout) = self .patch_sandbox_operating_state(sandbox_id, false) .await?; let stopped = self @@ -1779,12 +1801,21 @@ impl KubernetesComputeDriver { ) .await; // Scale the paired supervisor down only once the workload has actually - // stopped, so a graceful shutdown that needs egress still has it. + // stopped, so a graceful shutdown that needs egress still has it. This + // is best-effort: the workload is already stopped, so a failed + // scale-down only wastes supervisor resources and must not fail the + // stop. A later start or delete reconciles the replica count. if stopped.is_ok() - && let Some(sandbox_name) = sandbox_name.as_deref() + && let Err(err) = self + .scale_proxy_pod_supervisor(&kube_name, &namespace, 0) + .await { - self.scale_proxy_pod_supervisor(sandbox_name, &namespace, 0) - .await; + warn!( + sandbox_id = %sandbox_id, + cr_name = %kube_name, + error = %err, + "Failed to scale proxy-pod supervisor down on stop" + ); } stopped } @@ -1849,12 +1880,13 @@ impl KubernetesComputeDriver { } pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let (_api, _kube_name, sandbox_name, _pod_name, namespace, _timeout) = + let (_api, kube_name, _pod_name, namespace, _timeout) = self.patch_sandbox_operating_state(sandbox_id, true).await?; - if let Some(sandbox_name) = sandbox_name.as_deref() { - self.scale_proxy_pod_supervisor(sandbox_name, &namespace, 1) - .await; - } + // Propagate scale-up failure: the agent pod cannot itself retry a + // Deployment scale, so a swallowed error would wedge the sandbox in + // Starting with a supervisor stuck at zero replicas. + self.scale_proxy_pod_supervisor(&kube_name, &namespace, 1) + .await?; Ok(()) } @@ -1862,17 +1894,7 @@ impl KubernetesComputeDriver { &self, sandbox_id: &str, running: bool, - ) -> Result< - ( - AgentSandboxApi, - String, - Option, - String, - String, - Duration, - ), - KubernetesDriverError, - > { + ) -> Result<(AgentSandboxApi, String, String, String, Duration), KubernetesDriverError> { let lookup_api = self .supported_sandbox_api_for_lookup(self.client.clone()) .await @@ -1897,9 +1919,6 @@ impl KubernetesComputeDriver { .into_iter() .next() .ok_or(KubernetesDriverError::NotFound)?; - // Proxy-pod companion resources are named from the sandbox name, which - // is not the CR name. - let sandbox_name = annotation_or_label(&object, LABEL_SANDBOX_NAME); let namespace = object .metadata .namespace @@ -1953,7 +1972,6 @@ impl KubernetesComputeDriver { Ok(( agent_sandbox_api, kube_name, - sandbox_name, pod_name, namespace, stop_timeout, @@ -1972,79 +1990,78 @@ impl KubernetesComputeDriver { .await?; let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); - let (kube_name, sandbox_name, obj_namespace, _workspace, preconditions) = - match tokio::time::timeout(KUBE_API_TIMEOUT, lookup_api.api.list(&lp)).await { - Ok(Ok(list)) => { - if let Some(obj) = list.items.into_iter().next() { - // Per-sandbox proxy-pod resources are named from the - // sandbox name, not the CR name. They differ: a CR is - // `--`. - let sandbox_name = annotation_or_label(&obj, LABEL_SANDBOX_NAME); - match obj.metadata.name { - Some(name) => { - let ns = obj - .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - let ws = obj - .metadata - .labels - .as_ref() - .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) - .unwrap_or_default(); - let pc = Preconditions { - uid: obj.metadata.uid, - resource_version: obj.metadata.resource_version, - }; - (name, sandbox_name, ns, ws, pc) - } - None => return Ok(false), + let (kube_name, obj_namespace, _workspace, preconditions) = match tokio::time::timeout( + KUBE_API_TIMEOUT, + lookup_api.api.list(&lp), + ) + .await + { + Ok(Ok(list)) => { + if let Some(obj) = list.items.into_iter().next() { + match obj.metadata.name { + Some(name) => { + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let ws = obj + .metadata + .labels + .as_ref() + .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) + .unwrap_or_default(); + let pc = Preconditions { + uid: obj.metadata.uid, + resource_version: obj.metadata.resource_version, + }; + (name, ns, ws, pc) } - } else { - debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)"); - return Ok(false); + None => return Ok(false), } + } else { + debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)"); + return Ok(false); } - Ok(Err(err)) => { - warn!( - sandbox_id = %sandbox_id, - error = %err, - "Failed to list sandbox for deletion from Kubernetes" - ); - return Err(err.to_string()); - } - Err(_elapsed) => { - warn!( - sandbox_id = %sandbox_id, - timeout_secs = KUBE_API_TIMEOUT.as_secs(), - "Timed out listing sandbox for deletion from Kubernetes" - ); - return Err(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() - )); - } - }; - - if self.config.topology == SupervisorTopology::ProxyPod { - if let Some(sandbox_name) = sandbox_name.as_deref() { - self.cleanup_proxy_pod_resources(sandbox_name, &obj_namespace) - .await; - } else { + } + Ok(Err(err)) => { warn!( sandbox_id = %sandbox_id, - kube_name = %kube_name, - "Sandbox CR has no sandbox-name label; leaving proxy-pod resources to owner-reference GC" + error = %err, + "Failed to list sandbox for deletion from Kubernetes" ); + return Err(err.to_string()); } - } + Err(_elapsed) => { + warn!( + sandbox_id = %sandbox_id, + timeout_secs = KUBE_API_TIMEOUT.as_secs(), + "Timed out listing sandbox for deletion from Kubernetes" + ); + return Err(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )); + } + }; let delete_api = self .supported_agent_sandbox_api(self.client.clone(), &obj_namespace) .await?; let dp = DeleteParams::default().preconditions(preconditions); - match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { + // Delete the Sandbox CR (which tears down the workload pod) BEFORE + // removing the proxy-pod companion resources. The agent egress + // NetworkPolicy is the egress fence; removing it while the workload is + // still running would open unrestricted egress, and if the CR delete + // failed the exposure would persist. Companions are owner-referenced to + // the CR, so this ordering also matches Kubernetes garbage collection; + // the explicit cleanup below only accelerates it. + let deleted = match tokio::time::timeout( + KUBE_API_TIMEOUT, + delete_api.api.delete(&kube_name, &dp), + ) + .await + { Ok(Ok(_response)) => { info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); Ok(true) @@ -2072,7 +2089,15 @@ impl KubernetesComputeDriver { KUBE_API_TIMEOUT.as_secs() )) } + }; + + // Only remove the egress fence once the CR (and its workload) deletion + // has been initiated. On CR-delete failure the fence stays in place. + if deleted.is_ok() && self.config.topology == SupervisorTopology::ProxyPod { + self.cleanup_proxy_pod_resources(&kube_name, &obj_namespace) + .await; } + deleted } pub async fn sandbox_exists(&self, sandbox_id: &str) -> Result { @@ -2703,8 +2728,6 @@ const LABEL_SANDBOX_ROLE: &str = "openshell.ai/sandbox-role"; const SANDBOX_ROLE_AGENT: &str = "agent"; const SANDBOX_ROLE_SUPERVISOR: &str = "supervisor"; const PROXY_POD_PROXY_PORT: u16 = 3128; -const PROXY_POD_GATEWAY_FORWARD_PORT: u16 = 18080; -const PROXY_POD_GATEWAY_FORWARD_ADDR: &str = "0.0.0.0:18080"; const PROXY_POD_WAIT_INIT_CONTAINER_NAME: &str = "openshell-wait-for-proxy"; /// Upper bound on how long the agent pod waits for its paired supervisor. /// Exceeding it fails the init container, which surfaces as a pod-level error @@ -4830,11 +4853,6 @@ fn proxy_pod_supervisor_env( openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, "relaxed", ); - upsert_env( - &mut env, - openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR, - PROXY_POD_GATEWAY_FORWARD_ADDR, - ); upsert_env( &mut env, openshell_core::sandbox_env::PROXY_BIND_ADDR, @@ -4922,12 +4940,6 @@ fn proxy_pod_supervisor_service( "port": PROXY_POD_PROXY_PORT, "targetPort": PROXY_POD_PROXY_PORT, "protocol": "TCP" - }, - { - "name": "gateway-forward", - "port": PROXY_POD_GATEWAY_FORWARD_PORT, - "targetPort": PROXY_POD_GATEWAY_FORWARD_PORT, - "protocol": "TCP" } ] } @@ -4939,6 +4951,7 @@ fn proxy_pod_supervisor_deployment( template_environment: &std::collections::HashMap, spec_environment: &std::collections::HashMap, params: &SandboxPodParams<'_>, + pod_config: &KubernetesPodDriverConfig, owner_ref: serde_json::Value, ) -> Deployment { let mut container = serde_json::json!({ @@ -4950,8 +4963,7 @@ fn proxy_pod_supervisor_deployment( ], "env": proxy_pod_supervisor_env(template_environment, spec_environment, params), "ports": [ - {"name": "http-proxy", "containerPort": PROXY_POD_PROXY_PORT, "protocol": "TCP"}, - {"name": "gateway-fwd", "containerPort": PROXY_POD_GATEWAY_FORWARD_PORT, "protocol": "TCP"} + {"name": "http-proxy", "containerPort": PROXY_POD_PROXY_PORT, "protocol": "TCP"} ], "readinessProbe": { "tcpSocket": {"port": PROXY_POD_PROXY_PORT}, @@ -5076,6 +5088,9 @@ fn proxy_pod_supervisor_deployment( } })); } + if let Some(spec_obj) = spec.as_object_mut() { + apply_pod_driver_config(spec_obj, pod_config); + } k8s_object(serde_json::json!({ "apiVersion": "apps/v1", @@ -5163,8 +5178,7 @@ fn proxy_pod_agent_egress_network_policy( } }], "ports": [ - {"protocol": "TCP", "port": PROXY_POD_PROXY_PORT}, - {"protocol": "TCP", "port": PROXY_POD_GATEWAY_FORWARD_PORT} + {"protocol": "TCP", "port": PROXY_POD_PROXY_PORT} ] })]; egress.extend(proxy_pod_dns_egress_rules(params.proxy_pod_dns_peers)); @@ -5214,8 +5228,7 @@ fn proxy_pod_supervisor_ingress_network_policy( } }], "ports": [ - {"protocol": "TCP", "port": PROXY_POD_PROXY_PORT}, - {"protocol": "TCP", "port": PROXY_POD_GATEWAY_FORWARD_PORT} + {"protocol": "TCP", "port": PROXY_POD_PROXY_PORT} ] }] } @@ -7715,6 +7728,7 @@ mod tests { &std::collections::HashMap::new(), &std::collections::HashMap::new(), ¶ms, + &KubernetesPodDriverConfig::default(), owner_ref.clone(), )) .unwrap(); @@ -7753,10 +7767,6 @@ mod tests { rendered_env(container, openshell_core::sandbox_env::PROXY_BIND_ADDR), Some("0.0.0.0:3128") ); - assert_eq!( - rendered_env(container, openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR), - Some(PROXY_POD_GATEWAY_FORWARD_ADDR) - ); let agent_egress = serde_json::to_value(proxy_pod_agent_egress_network_policy( &names, @@ -7881,20 +7891,53 @@ mod tests { /// from the CR name silently targets objects that do not exist, which /// owner-reference GC then masks on delete but not on stop/start. #[test] - fn proxy_pod_resource_names_come_from_the_sandbox_name_not_the_cr_name() { - let from_sandbox_name = proxy_pod_resource_names("rdy"); - let from_cr_name = proxy_pod_resource_names("default--rdy"); - + fn proxy_pod_supervisor_inherits_workload_node_placement() { + let names = proxy_pod_resource_names("ws--dev"); + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_image: "supervisor:latest", + namespace: "agents", + sandbox_id: "sandbox-1", + sandbox_name: "dev", + proxy_uid: 2000, + sandbox_uid: 1500, + sandbox_gid: 1500, + ..SandboxPodParams::default() + }; + let pod_config = KubernetesPodDriverConfig { + node_selector: std::iter::once(("pool".to_string(), "gpu".to_string())).collect(), + tolerations: vec![serde_json::json!({"key": "gpu", "operator": "Exists"})], + ..KubernetesPodDriverConfig::default() + }; + let dep = serde_json::to_value(proxy_pod_supervisor_deployment( + &names, + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + ¶ms, + &pod_config, + serde_json::json!({}), + )) + .unwrap(); + let pod_spec = &dep["spec"]["template"]["spec"]; + assert_eq!(pod_spec["nodeSelector"]["pool"], "gpu"); + assert_eq!(pod_spec["tolerations"][0]["key"], "gpu"); + } + + #[test] + fn proxy_pod_resource_names_disambiguate_by_cr_name() { + // In shared mode two workspaces may hold a sandbox named `dev`, giving + // CR names `workspace-a--dev` and `workspace-b--dev`. Companion names + // must derive from the CR name so they do not collide; the bare + // sandbox name would. + let a = proxy_pod_resource_names("workspace-a--dev"); + let b = proxy_pod_resource_names("workspace-b--dev"); + assert_ne!(a.supervisor_deployment, b.supervisor_deployment); + assert_ne!(a.service, b.service); + assert_ne!(a.proxy_ca_secret, b.proxy_ca_secret); + assert_ne!(a.agent_egress_network_policy, b.agent_egress_network_policy); assert_ne!( - from_sandbox_name.supervisor_deployment, - from_cr_name.supervisor_deployment - ); - assert!( - from_sandbox_name - .supervisor_deployment - .starts_with("os-sup-rdy-"), - "{}", - from_sandbox_name.supervisor_deployment + a.supervisor_ingress_network_policy, + b.supervisor_ingress_network_policy ); } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index c8cb395a17..5106ab36d5 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -65,8 +65,6 @@ use openshell_supervisor_network::opa::OpaEngine; use openshell_supervisor_process::process::ProcessEnforcementMode; pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; use openshell_supervisor_process::skills; -use tokio::io::copy_bidirectional; -use tokio::net::{TcpListener, TcpStream}; use tokio::sync::mpsc::UnboundedSender; #[cfg(any(test, target_os = "linux"))] use tokio::time::timeout; @@ -553,20 +551,6 @@ pub async fn run_sandbox( None }; - let _gateway_forward = if network_enabled && proxy_pod_network_enforcement { - if !matches!(policy.network.mode, NetworkMode::Proxy) { - return Err(miette::miette!( - "external network enforcement requires proxy network mode" - )); - } - let endpoint = openshell_endpoint_for_proxy.as_deref().ok_or_else(|| { - miette::miette!("proxy-pod network enforcement requires an OpenShell gateway endpoint") - })?; - Some(start_gateway_forward_from_env(endpoint).await?) - } else { - None - }; - #[cfg(target_os = "linux")] let sidecar_control_server = if network_enabled && sidecar_network_enforcement { if !matches!(policy.network.mode, NetworkMode::Proxy) { @@ -1322,100 +1306,6 @@ fn process_policy_for_topology( Ok(process_policy) } -struct GatewayForwardHandle { - task: tokio::task::JoinHandle<()>, -} - -impl Drop for GatewayForwardHandle { - fn drop(&mut self) { - self.task.abort(); - } -} - -async fn start_gateway_forward_from_env(endpoint: &str) -> Result { - let listen_addr = - std::env::var(openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR).map_err(|_| { - miette::miette!( - "{} is required for proxy-pod gateway forwarding", - openshell_core::sandbox_env::GATEWAY_FORWARD_ADDR - ) - })?; - start_gateway_forward(&listen_addr, endpoint).await -} - -async fn start_gateway_forward(listen_addr: &str, endpoint: &str) -> Result { - let upstream = gateway_tcp_addr(endpoint)?; - let listener = TcpListener::bind(listen_addr).await.into_diagnostic()?; - info!( - listen_addr, - upstream, "Gateway TCP forward started for proxy-pod topology" - ); - - let task = tokio::spawn(async move { - loop { - let (mut inbound, peer) = match listener.accept().await { - Ok(accepted) => accepted, - Err(e) => { - warn!(error = %e, "Gateway forward accept failed"); - continue; - } - }; - let upstream = upstream.clone(); - tokio::spawn(async move { - let mut outbound = match TcpStream::connect(&upstream).await { - Ok(stream) => stream, - Err(e) => { - warn!(peer = %peer, upstream, error = %e, "Gateway forward connect failed"); - return; - } - }; - if let Err(e) = copy_bidirectional(&mut inbound, &mut outbound).await { - debug!(peer = %peer, error = %e, "Gateway forward connection closed with error"); - } - }); - } - }); - - Ok(GatewayForwardHandle { task }) -} - -fn gateway_tcp_addr(endpoint: &str) -> Result { - let (scheme, rest) = endpoint - .split_once("://") - .ok_or_else(|| miette::miette!("gateway endpoint must include a URL scheme"))?; - let default_port = match scheme { - "http" => 80, - "https" => 443, - other => { - return Err(miette::miette!( - "unsupported gateway endpoint scheme '{other}' for proxy-pod forwarding" - )); - } - }; - let authority = rest.split('/').next().unwrap_or(rest); - if authority.is_empty() { - return Err(miette::miette!("gateway endpoint is missing a host")); - } - if authority.starts_with('[') { - let closing = authority - .find(']') - .ok_or_else(|| miette::miette!("invalid bracketed IPv6 gateway endpoint"))?; - let host = &authority[..=closing]; - let port = authority[closing + 1..] - .strip_prefix(':') - .and_then(|value| value.parse::().ok()) - .unwrap_or(default_port); - return Ok(format!("{host}:{port}")); - } - let (host, port) = match authority.rsplit_once(':') { - Some((host, port)) if !host.is_empty() => { - (host, port.parse::().unwrap_or(default_port)) - } - _ => (authority, default_port), - }; - Ok(format!("{host}:{port}")) -} - /// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. async fn flush_proposals_to_gateway( endpoint: &str, diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index eb1ed8e1d0..38e062e3b1 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -146,4 +146,51 @@ rules: - patch - update {{- end }} + {{- if eq (.Values.supervisor.topology | default "combined") "proxy-pod" }} + # Proxy-pod topology creates a supervisor Deployment, Service, CA Secret, and + # NetworkPolicy pair per sandbox in the sandbox's namespace. In managed and + # operator modes that namespace is per-workspace, so these permissions must be + # cluster-scoped. `patch` on deployments scales the supervisor on stop/start, + # and `get` on replicasets lets the gateway verify the supervisor pod's + # Pod -> ReplicaSet -> Deployment -> Sandbox owner chain during ServiceAccount + # bootstrap. + - apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - watch + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - apiGroups: + - "" + resources: + - services + - secrets + verbs: + - create + - delete + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - watch + {{- end }} {{- end }} From 3e83dd7b04383d0fad5162f50d6fe4d78d93e6f4 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 20:58:21 -0400 Subject: [PATCH 21/25] fix(server): release the sessionless marker when a sandbox is removed set_sessionless recorded proxy-pod sandboxes in the supervisor session registry, but forget_sessionless was never called, so the set grew without bound as ephemeral proxy-pod sandboxes were created and deleted. Clear the marker in cleanup_sandbox_state, which runs on permanent removal. It is deliberately not cleared in the stopped-session cleanup: the sessionless property is a topology fact that must survive stop/start. Signed-off-by: Russell Bryant --- crates/openshell-server/src/compute/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index a09e4f021b..d8e3e08e5a 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -3131,6 +3131,10 @@ impl ComputeRuntime { self.tracing_log_bus.remove(sandbox_id); self.tracing_log_bus.platform_event_bus.remove(sandbox_id); self.sandbox_watch_bus.remove(sandbox_id); + // Drop the sessionless marker on permanent removal only. It is a + // topology property that must survive stop/start, so it is not cleared + // in cleanup_stopped_sandbox_sessions. + self.supervisor_sessions.forget_sessionless(sandbox_id); } async fn reconcile_snapshot_sandbox( From c7c1edea7f9532d167ef8b18a9f64333ef5e03e1 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 20:58:37 -0400 Subject: [PATCH 22/25] fix(cli): detect a sessionless topology before session-bound create steps The sessionless check ran after the structured-output early return and after the upload, forward, and editor steps. So sandbox create --output json -- exited zero while proxy-pod silently discarded the command, and upload, forward, or editor failures on --no-keep bypassed cleanup and leaked the ephemeral sandbox. Detect the sessionless topology at the top of the Ready arm. When a session-requiring operation was requested (a command, upload, forward, or editor), abort immediately -- cleaning up an ephemeral sandbox and reporting a non-zero exit -- before any of those steps or a structured-success print. A bare create with no such operation still succeeds (the network-only sandbox is created), emitting structured output when requested and detaching otherwise. Signed-off-by: Russell Bryant --- crates/openshell-cli/src/run.rs | 52 +++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 3e56571075..560be30360 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -958,6 +958,31 @@ pub async fn sandbox_create( drop(stream); drop(client); + // Detect a sessionless topology (e.g. proxy-pod) before any + // operation that needs a supervisor session. Uploads, port + // forwarding, the editor, an exec command, and interactive connect + // all require one. Handling this first means a discarded command is + // never reported as success (including with --output json) and an + // ephemeral (--no-keep) sandbox is cleaned up rather than leaked. + let sessionless = sandbox_has_no_supervisor_session(&last_sandbox); + if sessionless + && (!command.is_empty() + || !uploads.is_empty() + || forward.is_some() + || editor.is_some()) + { + return Err(abort_sessionless_create( + &effective_server, + &sandbox_name, + persist, + workspace, + &effective_tls, + gateway_name, + !command.is_empty(), + ) + .await); + } + let upload_count = uploads.len(); for (idx, (local_path, sandbox_path, git_ignore)) in uploads.iter().enumerate() { let dest = sandbox_path.as_deref(); @@ -1065,25 +1090,6 @@ pub async fn sandbox_create( return Ok(()); } - let sessionless = sandbox_has_no_supervisor_session(&last_sandbox); - - // A command given to a sessionless topology never runs: there is no - // supervisor to launch it and no session to exec it. Surface that - // even in the implicit-detach path a non-interactive persistent - // create would otherwise take silently. - if sessionless && !command.is_empty() { - return Err(abort_sessionless_create( - &effective_server, - &sandbox_name, - persist, - workspace, - &effective_tls, - gateway_name, - true, - ) - .await); - } - // Persistent non-interactive creates detach implicitly. An // explicitly ephemeral (`--no-keep`) create must still attach so // it can observe the canonical process and delete the sandbox when @@ -1095,9 +1101,11 @@ pub async fn sandbox_create( return Ok(()); } - // An interactive create against a sessionless topology cannot - // attach. Skip the session — which would spawn ssh, fail inside the - // subprocess, and surface as an opaque exit status — and explain. + // An interactive bare create against a sessionless topology cannot + // attach (session-requiring operations were already rejected at the + // top of this arm). Skip the connect — which would spawn ssh, fail + // inside the subprocess, and surface as an opaque exit status — and + // explain instead. if sessionless { return Err(abort_sessionless_create( &effective_server, From c5c3af9a6ccd8648d2b032a34b8752c361981f59 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 20:58:51 -0400 Subject: [PATCH 23/25] test(e2e): add a capability-scoped proxy-pod suite The e2e:kubernetes:proxy-pod task ran the generic Kubernetes suite, whose smoke test execs a command and reads its output -- capabilities proxy-pod lacks, so the suite could not pass. Add tests/proxy_pod.rs (feature e2e-kubernetes-proxy-pod) covering the topology's actual contract: a workload whose entrypoint is set through containers.agent.command reaches Ready, and relay-backed operations (exec) are rejected with a topology-specific error. Scope the task to this suite instead of the incompatible generic one. The NetworkPolicy egress boundary is asserted at the unit level in the driver and validated manually on a policy-enforcing cluster; a self-probing egress e2e needs a workload image that tests its own egress and reports through 'openshell logs', tracked as follow-up. Signed-off-by: Russell Bryant --- e2e/rust/Cargo.toml | 6 ++ e2e/rust/tests/proxy_pod.rs | 139 ++++++++++++++++++++++++++++++++++++ tasks/test.toml | 7 +- 3 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 e2e/rust/tests/proxy_pod.rs diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 0b8b56f3e5..92605f5266 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -31,6 +31,7 @@ e2e-kubernetes = ["e2e"] e2e-kubernetes-credential-drivers = ["e2e-kubernetes"] e2e-kubernetes-workspace-managed = ["e2e-kubernetes"] e2e-kubernetes-workspace-operator = ["e2e-kubernetes"] +e2e-kubernetes-proxy-pod = ["e2e-kubernetes"] e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"] e2e-podman-gpu = ["e2e-podman", "e2e-gpu"] e2e-oidc-pkce = [] @@ -106,6 +107,11 @@ name = "kubernetes_corporate_proxy" path = "tests/kubernetes_corporate_proxy.rs" required-features = ["e2e-kubernetes"] +[[test]] +name = "proxy_pod" +path = "tests/proxy_pod.rs" +required-features = ["e2e-kubernetes-proxy-pod"] + [[test]] name = "credential_drivers" path = "tests/credential_drivers.rs" diff --git a/e2e/rust/tests/proxy_pod.rs b/e2e/rust/tests/proxy_pod.rs new file mode 100644 index 0000000000..70aebc98be --- /dev/null +++ b/e2e/rust/tests/proxy_pod.rs @@ -0,0 +1,139 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes-proxy-pod")] + +//! Capability-scoped coverage for the Kubernetes `proxy-pod` topology. +//! +//! The generic Kubernetes suite (e.g. `smoke`) assumes an in-sandbox +//! supervisor: it execs a command and reads its captured output. `proxy-pod` +//! has no supervisor in the workload pod, so those tests cannot pass and are +//! not run for this topology. This suite instead verifies the contract +//! `proxy-pod` actually offers: +//! +//! - a workload whose entrypoint is set through `containers.agent.command` +//! reaches `Ready` (the canonical `-- ` path needs a supervisor and +//! does not apply here); +//! - relay-backed operations (`exec`) are rejected with a clear, +//! topology-specific error rather than hanging or failing opaquely. +//! +//! The NetworkPolicy egress boundary itself is asserted at the unit level in +//! `openshell-driver-kubernetes` (generated policy shape) and validated +//! manually on a policy-enforcing cluster; a self-probing egress e2e requires a +//! workload image that tests its own egress and reports through `openshell +//! logs`, which is tracked as follow-up. + +use std::process::Stdio; +use std::time::Duration; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::output::strip_ansi; + +/// Driver config that sets a long-running workload entrypoint. Required for +/// `proxy-pod`, whose image is run directly with no supervisor to launch a +/// canonical process. +const SLEEP_ENTRYPOINT: &str = + r#"{"kubernetes":{"containers":{"agent":{"command":["sleep","3600"]}}}}"#; + +/// Delete a sandbox by name, ignoring failures (best-effort cleanup). +async fn delete_sandbox(name: &str) { + let mut cmd = openshell_cmd(); + cmd.arg("sandbox") + .arg("delete") + .arg(name) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let _ = cmd.output().await; +} + +/// A workload whose entrypoint is supplied through driver config reaches +/// `Ready`, and relay-backed operations are rejected with a topology-specific +/// error. +#[tokio::test] +async fn proxy_pod_runs_workload_and_rejects_sessions() { + let name = "e2e-proxy-pod"; + // Best-effort cleanup from a previous interrupted run. + delete_sandbox(name).await; + + // Detached create: proxy-pod cannot open a session, so a non-detached + // create would report the sessionless topology instead of returning. + let mut create = openshell_cmd(); + create + .arg("sandbox") + .arg("create") + .arg("--name") + .arg(name) + .arg("--detach") + .arg("--driver-config-json") + .arg(SLEEP_ENTRYPOINT) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let create_out = tokio::time::timeout(Duration::from_secs(300), create.output()) + .await + .expect("sandbox create timed out") + .expect("failed to spawn openshell"); + let create_text = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&create_out.stdout), + String::from_utf8_lossy(&create_out.stderr), + )); + assert!( + create_out.status.success(), + "proxy-pod create with an entrypoint override should succeed:\n{create_text}", + ); + + // The sandbox should be present and reach Ready. + let mut ready = false; + let mut last_list = String::new(); + for _ in 0..30 { + let mut list = openshell_cmd(); + list.arg("sandbox") + .arg("list") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let out = list.output().await.expect("failed to run sandbox list"); + last_list = strip_ansi(&String::from_utf8_lossy(&out.stdout)); + if last_list + .lines() + .any(|line| line.contains(name) && line.contains("Ready")) + { + ready = true; + break; + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + assert!(ready, "proxy-pod sandbox never reached Ready:\n{last_list}"); + + // Relay-backed operations must fail fast with a topology-specific error, + // not hang or surface an opaque ssh failure. + let mut exec = openshell_cmd(); + exec.arg("sandbox") + .arg("exec") + .arg(name) + .arg("--") + .arg("echo") + .arg("hi") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let exec_out = tokio::time::timeout(Duration::from_secs(60), exec.output()) + .await + .expect("sandbox exec timed out") + .expect("failed to spawn openshell"); + let exec_text = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&exec_out.stdout), + String::from_utf8_lossy(&exec_out.stderr), + )); + assert!( + !exec_out.status.success(), + "exec against a sessionless topology must fail:\n{exec_text}", + ); + assert!( + exec_text.contains("no supervisor inside the sandbox") + || exec_text.contains("SSH, exec, port forwarding"), + "exec failure should name the topology limitation:\n{exec_text}", + ); + + delete_sandbox(name).await; +} diff --git a/tasks/test.toml b/tasks/test.toml index a431cc28dc..05a2db5ffd 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -161,8 +161,11 @@ env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sidec run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:proxy-pod"] -description = "Run Kubernetes e2e with the proxy-pod topology overlay; requires NetworkPolicy enforcement in the target cluster" -env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-proxy-pod.yaml" } +description = "Run the capability-scoped proxy-pod Kubernetes e2e suite; requires NetworkPolicy enforcement in the target cluster" +# proxy-pod is network-only: it has no in-sandbox supervisor, so the generic +# suite's exec/session tests (e.g. smoke) cannot pass. Run only the +# proxy_pod suite, which exercises this topology's actual contract. +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-proxy-pod.yaml", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e-kubernetes-proxy-pod", OPENSHELL_E2E_KUBE_TEST = "proxy_pod" } run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:db"] From 57e3bd7a28104496b59165bb08516f15235675ee Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 21:01:01 -0400 Subject: [PATCH 24/25] docs(rfc): reflect gateway-forward removal in proxy-pod design Update the topology diagram, NetworkPolicy contract, and validation table now that the supervisor no longer forwards a raw gateway tunnel, and note in the credential-isolation section that the workload has no network path to the gateway at all. Signed-off-by: Russell Bryant --- rfc/proxy-pod-topology-DRAFT.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/rfc/proxy-pod-topology-DRAFT.md b/rfc/proxy-pod-topology-DRAFT.md index b25e6ae840..b6413cadf0 100644 --- a/rfc/proxy-pod-topology-DRAFT.md +++ b/rfc/proxy-pod-topology-DRAFT.md @@ -21,7 +21,7 @@ originating issue before it moves out of draft. ## Summary This RFC proposes `proxy-pod`, a Kubernetes supervisor topology that moves -network enforcement and gateway forwarding out of the sandbox pod entirely and +network enforcement out of the sandbox pod entirely and into a paired, per-sandbox supervisor `Deployment`. The sandbox pod runs the agent image directly — no supervisor binary, no gateway credentials, no privileged init container, no shared process namespace. Egress is fenced by two @@ -130,7 +130,7 @@ flowchart TB Deployment["Supervisor Deployment
replicas: 1, owned by Sandbox CR"] subgraph SupervisorPod["Supervisor pod — role=supervisor"] - Proxy["openshell-supervisor --mode=network
:3128 proxy, :18080 gateway-fwd"] + Proxy["openshell-supervisor --mode=network
:3128 policy-enforced proxy"] end Service["Headless Service
clusterIP: None"] @@ -147,7 +147,6 @@ flowchart TB Deployment --> SupervisorPod AgentPod -->|"HTTP_PROXY / HTTPS_PROXY"| Service Service --> Proxy - Proxy -->|"gateway forwarding"| Gateway Proxy -->|"policy-enforced egress"| External CA -. mounted .- AgentPod CA -. mounted .- SupervisorPod @@ -217,6 +216,14 @@ control socket with peer-credential checks and one-shot listener semantics. `proxy-pod` has no such socket: the credential simply is not in the pod, and the two pods share no namespace, no filesystem, and no IPC. +The workload also has no network path to the gateway. Only the supervisor +connects to the gateway (for policy, inference, log push, and token bootstrap); +the agent egress `NetworkPolicy` permits the workload to reach only the +supervisor's proxy port and cluster DNS. An earlier revision ran a raw TCP +forward from the supervisor to the gateway that the workload could reach; it was +removed because nothing on the workload consumed it and, under unauthenticated +gateway access, it was a policy-bypassing path to the gateway API. + One consequence: because credentials are per-supervisor and the CA is generated per sandbox, a `proxy-pod` sandbox cannot participate in the corporate upstream-proxy credential feature, which mounts a `user:pass` Secret into the @@ -242,8 +249,8 @@ Two policies define the fence: **Agent egress** (`policyTypes: [Egress]`, selecting `sandbox-role=agent`) permits exactly two destinations: -1. Pods labeled `sandbox-role=supervisor` for this sandbox ID, on TCP 3128 and - TCP 18080. +1. Pods labeled `sandbox-role=supervisor` for this sandbox ID, on TCP 3128 (the + policy-enforced HTTP CONNECT proxy). 2. Cluster DNS, on UDP 53 and TCP 53. Everything else is denied. **This is load-bearing.** `HTTP_PROXY` is only a @@ -609,7 +616,7 @@ chart, then deployed to OpenShift 4.22.6 / OVN-Kubernetes. Measured results: | Agent resolves its paired supervisor `Service` | pass | | Direct egress to the internet denied | pass | | Direct egress to the gateway denied | pass | -| Egress to supervisor `:3128` / `:18080` allowed | pass | +| Egress to supervisor `:3128` allowed | pass | | Policy-denied host through the proxy | pass, 403 at CONNECT | | Policy-allowed host through the proxy | pass, HTTP 200 with the generated CA trusted | | All resources reclaimed on delete | pass | From 59e081bd7715a028249a259f2ae65650ca8fdc1f Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 21 Aug 2026 21:16:49 -0400 Subject: [PATCH 25/25] fix(kubernetes): reference proxy-pod companions by CR name in the pod template The companion resources are named from the Sandbox CR name, but the workload pod template still derived the CA secret mount and HTTP_PROXY Service name from the bare sandbox name, and the create-rollback path cleaned up bare-name resources in the static namespace. In shared mode the workload then mounted a CA secret that did not exist and never became Ready. Thread the CR name through SandboxPodParams and use it for the pod template's companion references and the rollback cleanup, matching create_proxy_pod_resources. Caught by re-testing a shared-mode create on the cluster. Signed-off-by: Russell Bryant --- .../openshell-driver-kubernetes/src/driver.rs | 65 ++++++++++++++++++- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 6a299f9054..df14b3dc92 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1421,6 +1421,7 @@ impl KubernetesComputeDriver { .resolve_sandbox_identity_in_namespace(&target_namespace) .await; + let cr_name = self.config.kube_resource_name(workspace, name); let params = SandboxPodParams { default_image: &self.config.default_image, image_pull_policy: &self.config.image_pull_policy, @@ -1451,6 +1452,7 @@ impl KubernetesComputeDriver { service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, sandbox_name: &sandbox.name, + cr_name: &cr_name, grpc_endpoint: &self.config.grpc_endpoint, ssh_socket_path: self.ssh_socket_path(), client_tls_secret_name: &self.config.client_tls_secret_name, @@ -1472,7 +1474,7 @@ impl KubernetesComputeDriver { let data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) .map_err(KubernetesDriverError::InvalidArgument)?; - let kube_name = self.config.kube_resource_name(workspace, name); + let kube_name = cr_name.clone(); let mut obj = DynamicObject::new(&kube_name, &agent_sandbox_api.resource); let mut annotations = sandbox_annotations(sandbox); for key in [ @@ -1548,7 +1550,7 @@ impl KubernetesComputeDriver { error = %err, "Failed to create proxy-pod resources; deleting Sandbox CR" ); - self.cleanup_proxy_pod_resources(name, &self.config.namespace) + self.cleanup_proxy_pod_resources(params.cr_name, params.namespace) .await; let _ = tokio::time::timeout( KUBE_API_TIMEOUT, @@ -3657,7 +3659,7 @@ fn apply_supervisor_proxy_pod_topology( apply_proxy_pod_affinity(spec, params.sandbox_id, params.proxy_pod_affinity); - let names = proxy_pod_resource_names(params.sandbox_name); + let names = proxy_pod_resource_names(params.cr_name); let service_dns = proxy_pod_service_dns(&names.service, params.namespace); let volumes = spec @@ -4016,6 +4018,10 @@ struct SandboxPodParams<'a> { service_account_name: &'a str, sandbox_id: &'a str, sandbox_name: &'a str, + /// Sandbox CR resource name (`kube_resource_name`), unique per sandbox in + /// every workspace mode. Companion resource names derive from this so they + /// match across the workload pod template and the companion objects. + cr_name: &'a str, grpc_endpoint: &'a str, ssh_socket_path: &'a str, client_tls_secret_name: &'a str, @@ -4060,6 +4066,7 @@ impl Default for SandboxPodParams<'_> { service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, sandbox_id: "", sandbox_name: "", + cr_name: "", grpc_endpoint: "", ssh_socket_path: "", client_tls_secret_name: "", @@ -7502,6 +7509,7 @@ mod tests { namespace: "agents", sandbox_id: "sandbox-123", sandbox_name: "example-sandbox", + cr_name: "example-sandbox", grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", proxy_uid: 2200, sandbox_uid: 1500, @@ -7618,6 +7626,7 @@ mod tests { namespace: "agents", sandbox_id: "sandbox-123", sandbox_name: "example-sandbox", + cr_name: "example-sandbox", proxy_uid: 2200, sandbox_uid: 1500, sandbox_gid: 1600, @@ -7706,6 +7715,7 @@ mod tests { service_account_name: "openshell-sandbox", sandbox_id: "sandbox-123", sandbox_name: "example-sandbox", + cr_name: "example-sandbox", grpc_endpoint: "http://openshell-gateway.openshell.svc:8080", proxy_uid: 2200, sandbox_uid: 1500, @@ -7843,6 +7853,7 @@ mod tests { namespace: "agents", sandbox_id: "sandbox-123", sandbox_name: "example-sandbox", + cr_name: "example-sandbox", proxy_pod_dns_peers: peers, ..SandboxPodParams::default() }; @@ -7923,6 +7934,52 @@ mod tests { assert_eq!(pod_spec["tolerations"][0]["key"], "gpu"); } + #[test] + fn proxy_pod_pod_template_references_companions_by_cr_name() { + // Shared mode: CR name is `--`, distinct from the bare + // sandbox name. The workload pod's CA secret mount and proxy URL must + // use the CR-name-derived companion names, or the pod mounts a secret + // that does not exist (and never becomes Ready). + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_image: "supervisor:latest", + namespace: "agents", + sandbox_id: "sandbox-1", + sandbox_name: "dev", + cr_name: "team-a--dev", + proxy_uid: 2000, + sandbox_uid: 1500, + sandbox_gid: 1500, + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent:latest".to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + let names = proxy_pod_resource_names("team-a--dev"); + let service_dns = proxy_pod_service_dns(&names.service, "agents"); + let agent = &pod_template["spec"]["containers"][0]; + assert_eq!( + rendered_env(agent, "HTTP_PROXY"), + Some(format!("http://{service_dns}:3128").as_str()) + ); + let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); + assert!( + volumes.iter().any(|v| { + v["name"] == "openshell-proxy-pod-ca-source" + && v["secret"]["secretName"] == serde_json::json!(names.proxy_ca_secret) + }), + "workload CA volume must reference the CR-name-derived secret {}", + names.proxy_ca_secret + ); + } + #[test] fn proxy_pod_resource_names_disambiguate_by_cr_name() { // In shared mode two workspaces may hold a sandbox named `dev`, giving @@ -7976,6 +8033,7 @@ mod tests { namespace: "agents", sandbox_id: "sandbox-123", sandbox_name: "example-sandbox", + cr_name: "example-sandbox", proxy_uid: 2200, sandbox_uid: 1500, sandbox_gid: 1500, @@ -8017,6 +8075,7 @@ mod tests { namespace: "agents", sandbox_id: "sandbox-123", sandbox_name: "example-sandbox", + cr_name: "example-sandbox", ..SandboxPodParams::default() }; let pod_template = sandbox_template_to_k8s(