From 45aea6f39b4b9d462dd6f653544a863863ded00c Mon Sep 17 00:00:00 2001 From: Kris Hicks Date: Mon, 17 Aug 2026 14:20:24 -0700 Subject: [PATCH] feat(docker): export driver traces over OTLP Mirror the VM and Podman driver tracing setup for Docker. Export Docker driver spans through OTLP/gRPC as the distinct openshell-driver-docker service, preserve gateway trace context, record lifecycle and asynchronous provisioning operations, and report gRPC failures. Docker currently runs in-process when selected as a built-in gateway driver. Add the same temporary server-boundary shim used by Podman so traces retain the shape they will have when Docker moves to a separate process. Generalize the gateway provider selection for both in-process drivers and share the OTLP collector fixture across Docker, Podman, and VM tracing tests. Signed-off-by: Kris Hicks --- AGENTS.md | 1 + Cargo.lock | 19 +- architecture/gateway.md | 4 +- crates/openshell-driver-docker/Cargo.toml | 9 +- crates/openshell-driver-docker/README.md | 6 + crates/openshell-driver-docker/src/lib.rs | 471 ++++++++++++++++-- .../src/otel_tracing.rs | 92 ++++ crates/openshell-driver-docker/src/tests.rs | 456 +++++++++++++++++ crates/openshell-driver-podman/Cargo.toml | 2 +- crates/openshell-driver-podman/src/grpc.rs | 2 +- .../src/otel_tracing.rs | 84 +--- crates/openshell-driver-vm/Cargo.toml | 2 +- .../openshell-driver-vm/src/otel_tracing.rs | 92 +--- crates/openshell-otel-test-support/Cargo.toml | 20 + crates/openshell-otel-test-support/src/lib.rs | 130 +++++ crates/openshell-otel/src/lib.rs | 39 +- crates/openshell-server/src/cli.rs | 2 +- crates/openshell-server/src/compute/mod.rs | 11 +- crates/openshell-server/src/otel_tracing.rs | 22 +- crates/openshell-server/src/tracing_setup.rs | 194 +++++--- docs/reference/gateway-config.mdx | 2 +- 21 files changed, 1368 insertions(+), 292 deletions(-) create mode 100644 crates/openshell-driver-docker/src/otel_tracing.rs create mode 100644 crates/openshell-otel-test-support/Cargo.toml create mode 100644 crates/openshell-otel-test-support/src/lib.rs diff --git a/AGENTS.md b/AGENTS.md index bd532c1971..28ec730c84 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,6 +38,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-gateway-interceptors/` | Gateway interceptors | Intercepts and transforms configured gRPC requests at the gateway routing boundary | | `crates/openshell-ocsf/` | OCSF logging | OCSF v1.7.0 event types, builders, shorthand/JSONL formatters, tracing layers | | `crates/openshell-otel/` | OpenTelemetry support | Shared OTLP trace provider, resource, and tracing-layer construction | +| `crates/openshell-otel-test-support/` | OpenTelemetry test support | Shared loopback OTLP collector fixture for tracing tests | | `crates/openshell-core/` | Shared core | Common types, configuration, error handling | | `crates/openshell-extension-core/` | Extension core | Shared extension identity, JWT claims, bearer-token rotation, and TLS transport primitives | | `crates/openshell-sdk/` | Shared client SDK | Async Rust gateway client (gRPC transport, TLS, OIDC refresh, edge tunnel); consumed by CLI, TUI, and `@openshell/sdk` | diff --git a/Cargo.lock b/Cargo.lock index a9eee33af9..c2b46ddc69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3819,6 +3819,10 @@ dependencies = [ "futures", "miette", "openshell-core", + "openshell-otel", + "openshell-otel-test-support", + "opentelemetry", + "opentelemetry_sdk", "prost-types", "serde", "serde_json", @@ -3830,6 +3834,7 @@ dependencies = [ "toml", "tonic", "tracing", + "tracing-opentelemetry", "tracing-subscriber", "url", ] @@ -3894,8 +3899,8 @@ dependencies = [ "nix 0.29.0", "openshell-core", "openshell-otel", + "openshell-otel-test-support", "opentelemetry", - "opentelemetry-proto", "opentelemetry_sdk", "prost-types", "rustix 1.1.4", @@ -3950,10 +3955,10 @@ dependencies = [ "oci-client", "openshell-core", "openshell-otel", + "openshell-otel-test-support", "openshell-policy", "openshell-vfio", "opentelemetry", - "opentelemetry-proto", "opentelemetry_sdk", "polling", "prost", @@ -4042,6 +4047,16 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "openshell-otel-test-support" +version = "0.0.0" +dependencies = [ + "opentelemetry-proto", + "tokio", + "tokio-stream", + "tonic", +] + [[package]] name = "openshell-policy" version = "0.0.0" diff --git a/architecture/gateway.md b/architecture/gateway.md index 32bca6a1f6..f7c80d1ed8 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -707,7 +707,9 @@ inbound request to provide a parent. gRPC status is recorded when response trailers arrive. The gateway forwards OTLP configuration and W3C trace context to managed -external drivers. Each driver exports under its own service name. +external drivers. Built-in drivers use dedicated in-process providers that +preserve the same RPC trace boundary. Each driver exports to the configured +collector under its own service name. Two invariants shape the failure behavior. Telemetry is diagnostic, so no OTLP failure stops the gateway from serving: a malformed endpoint is logged at diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 065b3ff4b6..1c9e675f77 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -16,12 +16,16 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } +openshell-otel = { path = "../openshell-otel" } +opentelemetry = { workspace = true } +tracing-opentelemetry = { workspace = true } tokio = { workspace = true } tonic = { workspace = true, features = ["transport"] } futures = { workspace = true } tokio-stream = { workspace = true } tracing = { workspace = true } +tracing-subscriber = { workspace = true } bytes = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -31,13 +35,16 @@ url = { workspace = true } clap = { workspace = true } miette = { workspace = true } toml = { workspace = true } -tracing-subscriber = { workspace = true } [dev-dependencies] +openshell-otel-test-support = { path = "../openshell-otel-test-support" } +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true, features = ["testing"] } prost-types = { workspace = true } tar = "0.4" temp-env = "0.3" tempfile = "3" +tracing-subscriber = { workspace = true } [lints] workspace = true diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index ac0d211582..17ae1cfa03 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -2,6 +2,12 @@ Docker-backed compute driver for local OpenShell gateways. +When the gateway configures `[openshell.gateway.otlp]`, Docker compute-driver +spans export to the same OTLP/gRPC collector with the service name +`openshell-driver-docker`. The in-process driver preserves the gateway trace +context and emits the compute-driver RPC boundary that a standalone driver +would expose. + The driver manages sandbox containers through the local Docker daemon with the `bollard` client. It is intended for developer environments where Docker is already available and running Kubernetes would be unnecessary. diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 33acf1a2c6..b1859f54cc 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -5,6 +5,8 @@ #![allow(clippy::result_large_err)] +pub mod otel_tracing; + use bollard::Docker; use bollard::errors::Error as BollardError; use bollard::models::{ @@ -55,17 +57,21 @@ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; use openshell_core::{Config, Error, Result as CoreResult}; +use opentelemetry::trace::TraceContextExt as _; use std::collections::{HashMap, HashSet}; +use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; +use std::task::{Context, Poll}; use std::time::Duration; use tokio::sync::{Mutex, broadcast, mpsc}; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; -use tracing::{debug, info, warn}; +use tracing::{Instrument as _, debug, info, warn}; +use tracing_opentelemetry::OpenTelemetrySpanExt as _; use url::Url; const WATCH_BUFFER: usize = 128; @@ -82,6 +88,28 @@ const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; const DOCKER_NETWORK_DRIVER: &str = "bridge"; +fn provisioning_span( + parent: &opentelemetry::Context, + sandbox: &DriverSandbox, + image_ref: &str, +) -> tracing::Span { + let span = tracing::info_span!( + parent: None, + "docker.provision", + otel.name = "docker.provision", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox.id, + sandbox.name = %sandbox.name, + image.ref = %image_ref, + ); + let parent_span_context = parent.span().span_context().clone(); + if parent_span_context.is_valid() { + let parent = opentelemetry::Context::new().with_remote_span_context(parent_span_context); + let _ = span.set_parent(parent); + } + span +} + /// Gateway-local configuration for the Docker compute driver. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] @@ -387,6 +415,119 @@ fn default_true() -> bool { type WatchStream = Pin> + Send + 'static>>; +struct TracedWatchStream { + inner: WatchStream, + span: tracing::Span, + finished: bool, +} + +impl Stream for TracedWatchStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let span = self.span.clone(); + let _entered = span.enter(); + let result = self.inner.as_mut().poll_next(cx); + if !self.finished { + match &result { + Poll::Ready(Some(Err(status))) => { + openshell_otel::mark_error(&self.span); + self.span + .record("rpc.grpc.status_code", status.code() as i32); + self.finished = true; + } + Poll::Ready(None) => { + self.span + .record("rpc.grpc.status_code", tonic::Code::Ok as i32); + self.finished = true; + } + Poll::Pending | Poll::Ready(Some(Ok(_))) => {} + } + } + result + } +} + +impl Drop for TracedWatchStream { + fn drop(&mut self) { + if !self.finished { + openshell_otel::mark_error(&self.span); + self.span + .record("rpc.grpc.status_code", tonic::Code::Cancelled as i32); + } + } +} + +/// Compute-driver service wrapper that preserves the standalone RPC trace +/// boundary while Docker runs in the gateway process. +#[derive(Clone)] +pub struct ComputeDriverService { + driver: DockerComputeDriver, + trace_in_process_rpc: bool, +} + +impl ComputeDriverService { + #[must_use] + pub fn new(driver: DockerComputeDriver) -> Self { + Self { + driver, + trace_in_process_rpc: false, + } + } + + #[must_use] + pub fn new_in_process(driver: DockerComputeDriver) -> Self { + Self { + driver, + trace_in_process_rpc: true, + } + } + + fn in_process_rpc_span( + &self, + operation: &'static str, + method: &'static str, + ) -> Option { + self.trace_in_process_rpc.then(|| { + tracing::info_span!( + target: "openshell_driver_docker::otel_tracing", + "driver_rpc", + otel.name = operation, + otel.kind = "server", + otel.status_code = tracing::field::Empty, + rpc.system = "grpc", + rpc.service = "openshell.compute.v1.ComputeDriver", + rpc.method = method, + rpc.grpc.status_code = tracing::field::Empty, + ) + }) + } + + async fn trace_rpc( + &self, + operation: &'static str, + method: &'static str, + future: impl Future>, + ) -> Result { + use tracing::Instrument as _; + + let Some(span) = self.in_process_rpc_span(operation, method) else { + return future.await; + }; + let result = future.instrument(span.clone()).await; + match &result { + Ok(_) => { + span.record("rpc.grpc.status_code", tonic::Code::Ok as i32); + } + Err(status) => { + openshell_otel::mark_error(&span); + span.record("rpc.grpc.status_code", status.code() as i32); + } + } + result + } +} + impl DockerComputeDriver { pub async fn new(config: &Config, docker_config: &DockerComputeConfig) -> CoreResult { let socket_path = docker_config @@ -728,7 +869,7 @@ impl DockerComputeDriver { &sandbox.id, "Scheduled", format!("Docker sandbox accepted for image \"{image}\""), - HashMap::from([("image_ref".to_string(), image)]), + HashMap::from([("image_ref".to_string(), image.clone())]), ); self.publish_sandbox_snapshot(pending_sandbox_snapshot( sandbox, @@ -740,9 +881,14 @@ impl DockerComputeDriver { let driver = self.clone(); let sandbox_for_task = sandbox.clone(); let sandbox_id = sandbox.id.clone(); - let task = tokio::spawn(async move { - driver.provision_sandbox(sandbox_for_task).await; - }); + let parent = tracing::Span::current().context(); + let provisioning_span = provisioning_span(&parent, sandbox, &image); + let task = tokio::spawn( + async move { + driver.provision_sandbox(sandbox_for_task).await; + } + .instrument(provisioning_span), + ); let mut pending = self.pending.lock().await; if let Some(record) = pending.get_mut(&sandbox_id) { @@ -765,20 +911,42 @@ impl DockerComputeDriver { } } + #[tracing::instrument( + name = "docker.provision_sandbox", + skip(self, sandbox), + fields( + otel.name = "docker.provision_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox.id, + sandbox.name = %sandbox.name, + ) + )] async fn provision_sandbox_inner( &self, sandbox: &DriverSandbox, ) -> Result<(), DockerProvisioningFailure> { + let span_status = openshell_otel::ErrorStatusGuard::current(); let validated = Self::validated_sandbox(sandbox, &self.config).map_err(|status| { DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) })?; let template = validated.template; - let image = self - .ensure_image_available(&sandbox.id, &template.image) - .await - .map_err(|status| { - DockerProvisioningFailure::new("ImagePullFailed", status.message()) - })?; + let image = async { + openshell_otel::record_error_result( + self.ensure_image_available(&sandbox.id, &template.image) + .await + .map_err(|status| { + DockerProvisioningFailure::new("ImagePullFailed", status.message()) + }), + ) + } + .instrument(tracing::info_span!( + "docker.prepare_image", + otel.name = "docker.prepare_image", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox.id, + image.ref = %template.image, + )) + .await?; let token_file_created = write_sandbox_token_file(sandbox, &self.config) .await .map_err(|status| { @@ -812,25 +980,37 @@ impl DockerComputeDriver { } DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) })?; - self.docker - .create_container( - Some( - CreateContainerOptionsBuilder::default() - .name(container_name.as_str()) - .build(), - ), - create_body, + async { + openshell_otel::record_error_result( + self.docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(container_name.as_str()) + .build(), + ), + create_body, + ) + .await + .map_err(|err| { + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + DockerProvisioningFailure::from_status( + "ContainerCreateFailed", + create_status_from_docker_error("create docker sandbox container", err), + ) + }), ) - .await - .map_err(|err| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::from_status( - "ContainerCreateFailed", - create_status_from_docker_error("create docker sandbox container", err), - ) - })?; + } + .instrument(tracing::info_span!( + "docker.create_container", + otel.name = "docker.create_container", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox.id, + container.name = %container_name, + )) + .await?; self.publish_docker_progress( &sandbox.id, "Created", @@ -838,7 +1018,20 @@ impl DockerComputeDriver { HashMap::from([("container_name".to_string(), container_name.clone())]), ); - if let Err(err) = self.docker.start_container(&container_name, None).await { + let start_result = async { + openshell_otel::record_error_result( + self.docker.start_container(&container_name, None).await, + ) + } + .instrument(tracing::info_span!( + "docker.start_container", + otel.name = "docker.start_container", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox.id, + container.name = %container_name, + )) + .await; + if let Err(err) = start_result { let cleanup = self .docker .remove_container( @@ -879,7 +1072,7 @@ impl DockerComputeDriver { ); } - Ok(()) + span_status.finish(Ok(())) } async fn delete_sandbox_inner( @@ -993,17 +1186,29 @@ impl DockerComputeDriver { /// Returns `Ok(true)` when a container existed and was started (or was /// already running), `Ok(false)` when no managed container is found for /// the sandbox, and `Err(...)` for any Docker failure. + #[tracing::instrument( + name = "docker.start_sandbox", + skip(self), + fields( + otel.name = "docker.start_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox_id, + sandbox.name = %sandbox_name, + ) + )] pub async fn start_sandbox( &self, sandbox_id: &str, sandbox_name: &str, ) -> Result { + let span_status = openshell_otel::ErrorStatusGuard::current(); + require_sandbox_identifier(sandbox_id, sandbox_name)?; self.lifecycle_event_fences.begin_start(sandbox_id); let result = self .start_sandbox_with_lifecycle_fence(sandbox_id, sandbox_name) .await; self.lifecycle_event_fences.finish_start(sandbox_id); - result + span_status.finish(result) } async fn start_sandbox_with_lifecycle_fence( @@ -1504,6 +1709,168 @@ impl DockerComputeDriver { } } +#[tonic::async_trait] +impl ComputeDriver for ComputeDriverService { + type WatchSandboxesStream = WatchStream; + + async fn get_capabilities( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.get_capabilities", + "get_capabilities", + ComputeDriver::get_capabilities(&self.driver, request), + ) + .await + } + + async fn get_gateway_listener_requirements( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.get_gateway_listener_requirements", + "get_gateway_listener_requirements", + ComputeDriver::get_gateway_listener_requirements(&self.driver, request), + ) + .await + } + + async fn validate_sandbox_create( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.validate_sandbox_create", + "validate_sandbox_create", + ComputeDriver::validate_sandbox_create(&self.driver, request), + ) + .await + } + + async fn get_sandbox( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.get_sandbox", + "get_sandbox", + ComputeDriver::get_sandbox(&self.driver, request), + ) + .await + } + + async fn list_sandboxes( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.list_sandboxes", + "list_sandboxes", + ComputeDriver::list_sandboxes(&self.driver, request), + ) + .await + } + + async fn create_sandbox( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.create_sandbox", + "create_sandbox", + ComputeDriver::create_sandbox(&self.driver, request), + ) + .await + } + + async fn stop_sandbox( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.stop_sandbox", + "stop_sandbox", + ComputeDriver::stop_sandbox(&self.driver, request), + ) + .await + } + + async fn start_sandbox( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.start_sandbox", + "start_sandbox", + ComputeDriver::start_sandbox(&self.driver, request), + ) + .await + } + + async fn delete_sandbox( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.delete_sandbox", + "delete_sandbox", + ComputeDriver::delete_sandbox(&self.driver, request), + ) + .await + } + + async fn watch_sandboxes( + &self, + request: Request, + ) -> Result, Status> { + use tracing::Instrument as _; + + let create_stream = ComputeDriver::watch_sandboxes(&self.driver, request); + let Some(span) = self.in_process_rpc_span("driver.watch_sandboxes", "watch_sandboxes") + else { + return create_stream.await; + }; + match create_stream.instrument(span.clone()).await { + Ok(response) => Ok(Response::new(Box::pin(TracedWatchStream { + inner: response.into_inner(), + span, + finished: false, + }))), + Err(status) => { + openshell_otel::mark_error(&span); + span.record("rpc.grpc.status_code", status.code() as i32); + Err(status) + } + } + } + + async fn ensure_workspace( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.ensure_workspace", + "ensure_workspace", + ComputeDriver::ensure_workspace(&self.driver, request), + ) + .await + } + + async fn delete_workspace( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.delete_workspace", + "delete_workspace", + ComputeDriver::delete_workspace(&self.driver, request), + ) + .await + } +} + #[tonic::async_trait] impl ComputeDriver for DockerComputeDriver { type WatchSandboxesStream = WatchStream; @@ -1590,22 +1957,44 @@ impl ComputeDriver for DockerComputeDriver { })) } + #[tracing::instrument( + name = "docker.schedule_sandbox", + skip(self, request), + fields( + otel.name = "docker.schedule_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %request.get_ref().sandbox.as_ref().map_or("", |sandbox| sandbox.id.as_str()), + sandbox.name = %request.get_ref().sandbox.as_ref().map_or("", |sandbox| sandbox.name.as_str()), + ) + )] async fn create_sandbox( &self, request: Request, ) -> Result, Status> { + let span_status = openshell_otel::ErrorStatusGuard::current(); let sandbox = request .into_inner() .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; self.create_sandbox_inner(&sandbox).await?; - Ok(Response::new(CreateSandboxResponse {})) + span_status.finish(Ok(Response::new(CreateSandboxResponse {}))) } + #[tracing::instrument( + name = "docker.stop_sandbox", + skip(self, request), + fields( + otel.name = "docker.stop_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %request.get_ref().sandbox_id, + sandbox.name = %request.get_ref().sandbox_name, + ) + )] async fn stop_sandbox( &self, request: Request, ) -> Result, Status> { + let span_status = openshell_otel::ErrorStatusGuard::current(); let request = request.into_inner(); require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; @@ -1613,7 +2002,7 @@ impl ComputeDriver for DockerComputeDriver { .await?; self.publish_container_snapshot(&request.sandbox_id, &request.sandbox_name) .await?; - Ok(Response::new(StopSandboxResponse {})) + span_status.finish(Ok(Response::new(StopSandboxResponse {}))) } async fn start_sandbox( @@ -1621,7 +2010,6 @@ impl ComputeDriver for DockerComputeDriver { request: Request, ) -> Result, Status> { let request = request.into_inner(); - require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; if !Self::start_sandbox(self, &request.sandbox_id, &request.sandbox_name).await? { return Err(Status::not_found("sandbox not found")); } @@ -1630,10 +2018,21 @@ impl ComputeDriver for DockerComputeDriver { Ok(Response::new(StartSandboxResponse {})) } + #[tracing::instrument( + name = "docker.delete_sandbox", + skip(self, request), + fields( + otel.name = "docker.delete_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %request.get_ref().sandbox_id, + sandbox.name = %request.get_ref().sandbox_name, + ) + )] async fn delete_sandbox( &self, request: Request, ) -> Result, Status> { + let span_status = openshell_otel::ErrorStatusGuard::current(); let request = request.into_inner(); require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; @@ -1652,7 +2051,7 @@ impl ComputeDriver for DockerComputeDriver { }); } - Ok(Response::new(DeleteSandboxResponse { deleted })) + span_status.finish(Ok(Response::new(DeleteSandboxResponse { deleted }))) } async fn watch_sandboxes( diff --git a/crates/openshell-driver-docker/src/otel_tracing.rs b/crates/openshell-driver-docker/src/otel_tracing.rs new file mode 100644 index 0000000000..9c388b5f04 --- /dev/null +++ b/crates/openshell-driver-docker/src/otel_tracing.rs @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! OpenTelemetry trace exporting for the Docker compute driver. + +use openshell_otel::{OtlpTraceConfig, SdkTracerProvider, ServiceName, SetupError}; +use tracing::Subscriber; +use tracing_subscriber::registry::LookupSpan; + +const SERVICE_NAME: &str = "openshell-driver-docker"; +const INSTRUMENTATION_SCOPE: &str = "openshell-driver-docker"; +pub const IN_PROCESS_TARGET_PREFIX: &str = "openshell_driver_docker"; + +#[must_use] +pub fn provider_for(endpoint: Option<&str>) -> (Option, Option) { + openshell_otel::provider_for(endpoint.map(|endpoint| OtlpTraceConfig { + endpoint, + service_name: ServiceName::Fixed(SERVICE_NAME), + service_version: Some(openshell_core::VERSION), + resource_attributes: Vec::new(), + })) +} + +pub fn layer(provider: &SdkTracerProvider) -> openshell_otel::OtlpLayer +where + S: Subscriber + for<'span> LookupSpan<'span>, +{ + openshell_otel::layer(provider, INSTRUMENTATION_SCOPE) +} + +pub fn in_process_layer(provider: &SdkTracerProvider) -> openshell_otel::TargetOtlpLayer +where + S: Subscriber + for<'span> LookupSpan<'span>, +{ + openshell_otel::layer_for_target_prefix( + provider, + INSTRUMENTATION_SCOPE, + IN_PROCESS_TARGET_PREFIX, + ) +} + +#[cfg(test)] +pub(crate) async fn test_lock() -> tokio::sync::MutexGuard<'static, ()> { + static LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + static INITIALIZED: std::sync::LazyLock<()> = std::sync::LazyLock::new(|| { + tracing::subscriber::set_global_default(tracing_subscriber::registry()) + .expect("test tracing subscriber installs once"); + }); + + let guard = LOCK.lock().await; + std::sync::LazyLock::force(&INITIALIZED); + guard +} + +#[cfg(test)] +mod tests { + use openshell_otel_test_support::OtlpTestServer; + use tracing_subscriber::layer::SubscriberExt as _; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn tracing_docker_driver_spans_reach_otlp_collector_with_distinct_service_name() { + let _tracing_lock = super::test_lock().await; + let collector = OtlpTestServer::start().await; + + let (provider, error) = super::provider_for(Some(collector.endpoint())); + assert!(error.is_none()); + let provider = provider.expect("provider"); + let subscriber = tracing_subscriber::registry().with(super::layer(&provider)); + tracing::subscriber::with_default(subscriber, || { + let span = tracing::info_span!("docker.schedule_sandbox", sandbox.id = "sb-otlp"); + drop(span.enter()); + drop(span); + }); + provider.force_flush().unwrap(); + collector.wait_for_export().await; + provider.shutdown().unwrap(); + let received = collector.shutdown().await; + + assert!( + received + .spans + .iter() + .any(|span| span.name == "docker.schedule_sandbox") + ); + assert!( + received + .service_names + .iter() + .any(|name| name == "openshell-driver-docker") + ); + } +} diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index f828120f63..b52cb87836 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -166,6 +166,462 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr } } +#[tokio::test] +async fn tracing_in_process_service_preserves_the_driver_rpc_server_boundary() { + use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; + use tracing::{Instrument as _, instrument::WithSubscriber as _}; + use tracing_subscriber::layer::SubscriberExt as _; + + let _tracing_lock = otel_tracing::test_lock().await; + let gateway_exporter = InMemorySpanExporterBuilder::new().build(); + let gateway_provider = SdkTracerProvider::builder() + .with_simple_exporter(gateway_exporter.clone()) + .build(); + let driver_exporter = InMemorySpanExporterBuilder::new().build(); + let driver_provider = SdkTracerProvider::builder() + .with_simple_exporter(driver_exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry() + .with(openshell_otel::layer_excluding_target_prefix( + &gateway_provider, + "gateway-test", + Some(otel_tracing::IN_PROCESS_TARGET_PREFIX), + )) + .with(otel_tracing::in_process_layer(&driver_provider)); + let service = ComputeDriverService::new_in_process(test_driver_with_config(runtime_config())); + + async { + let gateway_span = tracing::info_span!( + target: "openshell_server::compute", + "driver", + otel.name = "driver.get_capabilities", + otel.kind = "client" + ); + ComputeDriver::get_capabilities(&service, Request::new(GetCapabilitiesRequest {})) + .instrument(gateway_span) + .await?; + + let unrelated = tracing::info_span!( + target: "openshell_driver_kubernetes::compute", + "kubernetes.operation" + ); + drop(unrelated.enter()); + drop(unrelated); + Ok::<_, Status>(()) + } + .with_subscriber(subscriber) + .await + .expect("capabilities should succeed"); + async { + let gateway_span = tracing::info_span!( + target: "openshell_server::compute", + "driver", + otel.name = "driver.validate_sandbox_create", + otel.kind = "client" + ); + ComputeDriver::validate_sandbox_create( + &service, + Request::new(ValidateSandboxCreateRequest { sandbox: None }), + ) + .instrument(gateway_span) + .await + } + .with_subscriber( + tracing_subscriber::registry() + .with(openshell_otel::layer_excluding_target_prefix( + &gateway_provider, + "gateway-test", + Some(otel_tracing::IN_PROCESS_TARGET_PREFIX), + )) + .with(otel_tracing::in_process_layer(&driver_provider)), + ) + .await + .expect_err("missing sandbox should fail"); + gateway_provider.force_flush().unwrap(); + driver_provider.force_flush().unwrap(); + + let gateway_spans = gateway_exporter.get_finished_spans().unwrap(); + let driver_spans = driver_exporter.get_finished_spans().unwrap(); + let client = gateway_spans + .iter() + .find(|span| span.name == "driver.get_capabilities") + .unwrap(); + let server = driver_spans + .iter() + .find(|span| span.name == "driver.get_capabilities") + .expect("in-process server span"); + assert_eq!( + server.span_context.trace_id(), + client.span_context.trace_id() + ); + assert_eq!(server.parent_span_id, client.span_context.span_id()); + assert_eq!(server.span_kind, opentelemetry::trace::SpanKind::Server); + assert!(server.attributes.iter().any(|attribute| { + attribute.key.as_str() == "rpc.grpc.status_code" + && attribute.value.to_string() == (tonic::Code::Ok as i32).to_string() + })); + assert!( + gateway_spans + .iter() + .any(|span| span.name == "kubernetes.operation"), + "unrelated driver targets must remain gateway spans" + ); + assert!( + driver_spans + .iter() + .all(|span| span.name != "kubernetes.operation"), + "the Docker provider must not claim unrelated driver spans" + ); + let failed = driver_spans + .iter() + .find(|span| span.name == "driver.validate_sandbox_create") + .expect("failed in-process server span"); + assert!(matches!( + failed.status, + opentelemetry::trace::Status::Error { .. } + )); + assert!(failed.attributes.iter().any(|attribute| { + attribute.key.as_str() == "rpc.grpc.status_code" + && attribute.value.to_string() == (tonic::Code::InvalidArgument as i32).to_string() + })); + gateway_provider.shutdown().unwrap(); + driver_provider.shutdown().unwrap(); +} + +#[tokio::test] +async fn tracing_lifecycle_rpc_failures_export_docker_operation_spans() { + use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let _tracing_lock = otel_tracing::test_lock().await; + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(otel_tracing::layer(&provider)); + let driver = test_driver_with_config(runtime_config()); + + async { + ComputeDriver::create_sandbox( + &driver, + Request::new(CreateSandboxRequest { sandbox: None }), + ) + .await + .expect_err("missing sandbox should fail"); + ComputeDriver::start_sandbox(&driver, Request::new(StartSandboxRequest::default())) + .await + .expect_err("missing start identifier should fail"); + ComputeDriver::stop_sandbox(&driver, Request::new(StopSandboxRequest::default())) + .await + .expect_err("missing stop identifier should fail"); + ComputeDriver::delete_sandbox(&driver, Request::new(DeleteSandboxRequest::default())) + .await + .expect_err("missing delete identifier should fail"); + } + .with_subscriber(subscriber) + .await; + provider.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + for name in [ + "docker.schedule_sandbox", + "docker.start_sandbox", + "docker.stop_sandbox", + "docker.delete_sandbox", + ] { + let span = spans + .iter() + .find(|span| span.name == name) + .unwrap_or_else(|| panic!("{name} should be exported")); + assert!( + matches!(span.status, opentelemetry::trace::Status::Error { .. }), + "{name} should record the failed operation" + ); + } + provider.shutdown().unwrap(); +} + +#[tokio::test] +async fn tracing_direct_start_exports_a_docker_start_span() { + use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let _tracing_lock = otel_tracing::test_lock().await; + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(otel_tracing::layer(&provider)); + let driver = test_driver_with_config(runtime_config()); + + DockerComputeDriver::start_sandbox(&driver, "", "") + .with_subscriber(subscriber) + .await + .expect_err("missing identifier should fail"); + provider.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + let span = spans + .iter() + .find(|span| span.name == "docker.start_sandbox") + .expect("direct startup operation should export docker.start_sandbox"); + assert!(matches!( + span.status, + opentelemetry::trace::Status::Error { .. } + )); + provider.shutdown().unwrap(); +} + +#[tokio::test] +async fn tracing_image_preparation_failure_exports_nested_failed_spans() { + use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let _tracing_lock = otel_tracing::test_lock().await; + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(otel_tracing::layer(&provider)); + let mut config = runtime_config(); + config.image_pull_policy = "unsupported".to_string(); + let driver = test_driver_with_config(config); + + driver + .provision_sandbox_inner(&test_sandbox()) + .with_subscriber(subscriber) + .await + .expect_err("unsupported image pull policy should fail provisioning"); + provider.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + let provision = spans + .iter() + .find(|span| span.name == "docker.provision_sandbox") + .expect("provisioning span should be exported"); + assert!(matches!( + provision.status, + opentelemetry::trace::Status::Error { .. } + )); + let prepare_image = spans + .iter() + .find(|span| span.name == "docker.prepare_image") + .expect("image preparation span should be exported"); + assert_eq!( + prepare_image.parent_span_id, + provision.span_context.span_id() + ); + assert!(matches!( + prepare_image.status, + opentelemetry::trace::Status::Error { .. } + )); + provider.shutdown().unwrap(); +} + +#[tokio::test] +async fn background_provisioning_does_not_extend_the_scheduling_span_lifetime() { + use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; + use tracing::Instrument as _; + use tracing_opentelemetry::OpenTelemetrySpanExt as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let _tracing_lock = otel_tracing::test_lock().await; + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(otel_tracing::layer(&provider)); + let dispatch = tracing::Dispatch::new(subscriber); + let _dispatch = tracing::dispatcher::set_default(&dispatch); + + let scheduling = tracing::info_span!("docker.schedule_sandbox"); + let entered = scheduling.enter(); + let sandbox = test_sandbox(); + let provisioning = provisioning_span(&scheduling.context(), &sandbox, "test-image"); + let task = tokio::spawn(futures::future::pending::<()>().instrument(provisioning)); + drop(entered); + drop(scheduling); + provider.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + assert!( + spans + .iter() + .any(|span| span.name == "docker.schedule_sandbox"), + "the scheduling span should finish while background provisioning is pending" + ); + assert!( + spans.iter().all(|span| span.name != "docker.provision"), + "the provisioning span should remain open with the background task" + ); + + task.abort(); + task.await + .expect_err("the pending task should be cancelled"); + provider.shutdown().unwrap(); +} + +#[tokio::test] +async fn tracing_in_process_stream_span_lives_until_stream_failure() { + use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let _tracing_lock = otel_tracing::test_lock().await; + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(otel_tracing::in_process_layer(&provider)); + + async { + let span = tracing::info_span!( + target: "openshell_driver_docker::otel_tracing", + "driver_rpc", + otel.name = "driver.watch_sandboxes", + otel.kind = "server", + otel.status_code = tracing::field::Empty, + rpc.grpc.status_code = tracing::field::Empty, + ); + let inner: WatchStream = Box::pin(futures::stream::iter([Err(Status::internal( + "watch failed", + ))])); + let mut stream = TracedWatchStream { + inner, + span, + finished: false, + }; + + provider.force_flush().unwrap(); + assert!( + exporter.get_finished_spans().unwrap().is_empty(), + "server span must remain open while the response stream is alive" + ); + stream + .next() + .await + .expect("stream item") + .expect_err("stream should fail"); + drop(stream); + } + .with_subscriber(subscriber) + .await; + provider.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + let span = spans + .iter() + .find(|span| span.name == "driver.watch_sandboxes") + .expect("watch server span should be exported when the stream ends"); + assert!(matches!( + span.status, + opentelemetry::trace::Status::Error { .. } + )); + provider.shutdown().unwrap(); +} + +#[tokio::test] +async fn tracing_in_process_stream_records_ok_when_stream_completes() { + use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let _tracing_lock = otel_tracing::test_lock().await; + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(otel_tracing::in_process_layer(&provider)); + + async { + let span = tracing::info_span!( + target: "openshell_driver_docker::otel_tracing", + "driver_rpc", + otel.name = "driver.watch_sandboxes", + otel.kind = "server", + otel.status_code = tracing::field::Empty, + rpc.grpc.status_code = tracing::field::Empty, + ); + let inner: WatchStream = Box::pin(futures::stream::empty()); + let mut stream = TracedWatchStream { + inner, + span, + finished: false, + }; + + assert!(stream.next().await.is_none()); + drop(stream); + } + .with_subscriber(subscriber) + .await; + provider.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + let span = spans + .iter() + .find(|span| span.name == "driver.watch_sandboxes") + .expect("watch server span should be exported when the stream completes"); + assert!(span.attributes.iter().any(|attribute| { + attribute.key.as_str() == "rpc.grpc.status_code" + && attribute.value.to_string() == (tonic::Code::Ok as i32).to_string() + })); + provider.shutdown().unwrap(); +} + +#[tokio::test] +async fn tracing_in_process_stream_records_cancelled_when_dropped() { + use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let _tracing_lock = otel_tracing::test_lock().await; + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(otel_tracing::in_process_layer(&provider)); + + async { + let span = tracing::info_span!( + target: "openshell_driver_docker::otel_tracing", + "driver_rpc", + otel.name = "driver.watch_sandboxes", + otel.kind = "server", + otel.status_code = tracing::field::Empty, + rpc.grpc.status_code = tracing::field::Empty, + ); + let inner: WatchStream = Box::pin(futures::stream::pending()); + let stream = TracedWatchStream { + inner, + span, + finished: false, + }; + + drop(stream); + } + .with_subscriber(subscriber) + .await; + provider.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + let span = spans + .iter() + .find(|span| span.name == "driver.watch_sandboxes") + .expect("watch server span should be exported when the stream is cancelled"); + assert!(matches!( + span.status, + opentelemetry::trace::Status::Error { .. } + )); + assert!(span.attributes.iter().any(|attribute| { + attribute.key.as_str() == "rpc.grpc.status_code" + && attribute.value.to_string() == (tonic::Code::Cancelled as i32).to_string() + })); + provider.shutdown().unwrap(); +} + #[tokio::test] async fn gateway_listener_requirements_report_managed_bridge_address() { let config = runtime_config(); diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index 7442979711..8b3e014e8c 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -43,7 +43,7 @@ miette = { workspace = true } url = { workspace = true } [dev-dependencies] -opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "trace"] } +openshell-otel-test-support = { path = "../openshell-otel-test-support" } opentelemetry_sdk = { workspace = true, features = ["testing"] } prost-types = { workspace = true } temp-env = "0.3" diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index 7a7275a8a6..2cb59f086f 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -432,7 +432,7 @@ mod tests { .with(openshell_otel::layer_excluding_target_prefix( &gateway_provider, "gateway-test", - crate::otel_tracing::IN_PROCESS_TARGET_PREFIX, + Some(crate::otel_tracing::IN_PROCESS_TARGET_PREFIX), )) .with(crate::otel_tracing::in_process_layer(&driver_provider)); let service = ComputeDriverService::new_in_process(PodmanComputeDriver::for_tests( diff --git a/crates/openshell-driver-podman/src/otel_tracing.rs b/crates/openshell-driver-podman/src/otel_tracing.rs index 9d8f494276..51d42a88fe 100644 --- a/crates/openshell-driver-podman/src/otel_tracing.rs +++ b/crates/openshell-driver-podman/src/otel_tracing.rs @@ -132,59 +132,9 @@ pub(crate) async fn test_lock() -> tokio::sync::MutexGuard<'static, ()> { #[cfg(test)] mod tests { - use std::sync::{Arc, Mutex}; - - use opentelemetry_proto::tonic::collector::trace::v1::{ - ExportTraceServiceRequest, ExportTraceServiceResponse, - trace_service_server::{TraceService, TraceServiceServer}, - }; - use opentelemetry_proto::tonic::trace::v1::Span; + use openshell_otel_test_support::OtlpTestServer; use tracing_subscriber::layer::SubscriberExt as _; - #[derive(Default)] - struct Received { - spans: Vec, - service_names: Vec, - } - - #[derive(Clone)] - struct Collector { - received: Arc>, - exported: Arc, - } - - #[tonic::async_trait] - impl TraceService for Collector { - async fn export( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let mut received = self.received.lock().unwrap(); - for resource_span in request.into_inner().resource_spans { - if let Some(resource) = resource_span.resource { - received.service_names.extend( - resource - .attributes - .into_iter() - .filter(|attribute| attribute.key == "service.name") - .filter_map(|attribute| attribute.value) - .filter_map(|value| value.value) - .filter_map(|value| match value { - opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => Some(value), - _ => None, - }), - ); - } - for scope_span in resource_span.scope_spans { - received.spans.extend(scope_span.spans); - } - } - drop(received); - self.exported.notify_one(); - Ok(tonic::Response::new(ExportTraceServiceResponse::default())) - } - } - #[test] fn compute_driver_rpc_names_are_explicitly_mapped_and_schema_bounded() { for (rpc, operation, method) in [ @@ -243,28 +193,9 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn podman_driver_spans_reach_otlp_collector_with_distinct_service_name() { let _tracing_lock = super::test_lock().await; - let received = Arc::new(Mutex::new(Received::default())); - let exported = Arc::new(tokio::sync::Notify::new()); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let collector = Collector { - received: Arc::clone(&received), - exported: Arc::clone(&exported), - }; - let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); - let server = tokio::spawn(async move { - tonic::transport::Server::builder() - .add_service(TraceServiceServer::new(collector)) - .serve_with_incoming_shutdown( - tokio_stream::wrappers::TcpListenerStream::new(listener), - async { - let _ = shutdown_rx.await; - }, - ) - .await - }); + let collector = OtlpTestServer::start().await; - let (provider, error) = super::provider_for(Some(&format!("http://{address}"))); + let (provider, error) = super::provider_for(Some(collector.endpoint())); assert!(error.is_none()); let provider = provider.expect("provider"); let subscriber = tracing_subscriber::registry().with(super::layer(&provider)); @@ -273,16 +204,11 @@ mod tests { drop(span.enter()); drop(span); }); - let export_completed = exported.notified(); provider.force_flush().unwrap(); - tokio::time::timeout(std::time::Duration::from_secs(5), export_completed) - .await - .expect("OTLP export should complete"); + collector.wait_for_export().await; provider.shutdown().unwrap(); - shutdown_tx.send(()).unwrap(); - server.await.unwrap().unwrap(); + let received = collector.shutdown().await; - let received = received.lock().unwrap(); assert!( received .spans diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml index 0ee2740c77..ebcb9d2bc2 100644 --- a/crates/openshell-driver-vm/Cargo.toml +++ b/crates/openshell-driver-vm/Cargo.toml @@ -61,10 +61,10 @@ default = ["telemetry"] telemetry = ["openshell-core/telemetry"] [dev-dependencies] +openshell-otel-test-support = { path = "../openshell-otel-test-support" } temp-env = "0.3" tempfile = "3" opentelemetry_sdk = { workspace = true, features = ["testing"] } -opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "trace"] } # smol-rs/polling drives the BSD/macOS parent-death detection in # procguard via kqueue's EVFILT_PROC / NOTE_EXIT filter. We could use diff --git a/crates/openshell-driver-vm/src/otel_tracing.rs b/crates/openshell-driver-vm/src/otel_tracing.rs index 7ae75357db..e956e8310d 100644 --- a/crates/openshell-driver-vm/src/otel_tracing.rs +++ b/crates/openshell-driver-vm/src/otel_tracing.rs @@ -109,60 +109,9 @@ where #[cfg(test)] mod tests { - use std::sync::{Arc, Mutex}; - - use opentelemetry_proto::tonic::collector::trace::v1::{ - ExportTraceServiceRequest, ExportTraceServiceResponse, - trace_service_server::{TraceService, TraceServiceServer}, - }; - use opentelemetry_proto::tonic::trace::v1::Span; + use openshell_otel_test_support::OtlpTestServer; use tracing_subscriber::layer::SubscriberExt as _; - #[derive(Default)] - struct Received { - spans: Vec, - service_names: Vec, - } - - #[derive(Clone)] - struct Collector { - received: Arc>, - exported: Arc, - } - - #[tonic::async_trait] - impl TraceService for Collector { - async fn export( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - { - let mut received = self.received.lock().unwrap(); - for resource_span in request.into_inner().resource_spans { - if let Some(resource) = resource_span.resource { - received.service_names.extend( - resource - .attributes - .into_iter() - .filter(|attribute| attribute.key == "service.name") - .filter_map(|attribute| attribute.value) - .filter_map(|value| value.value) - .filter_map(|value| match value { - opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => Some(value), - _ => None, - }), - ); - } - for scope_span in resource_span.scope_spans { - received.spans.extend(scope_span.spans); - } - } - } - self.exported.notify_one(); - Ok(tonic::Response::new(ExportTraceServiceResponse::default())) - } - } - #[test] fn compute_driver_rpc_names_are_explicitly_mapped_and_schema_bounded() { for (rpc, operation, method) in [ @@ -212,28 +161,9 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn vm_driver_spans_reach_otlp_collector_with_distinct_service_name() { - let received = Arc::new(Mutex::new(Received::default())); - let exported = Arc::new(tokio::sync::Notify::new()); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let collector = Collector { - received: Arc::clone(&received), - exported: Arc::clone(&exported), - }; - let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); - let server = tokio::spawn(async move { - tonic::transport::Server::builder() - .add_service(TraceServiceServer::new(collector)) - .serve_with_incoming_shutdown( - tokio_stream::wrappers::TcpListenerStream::new(listener), - async { - let _ = shutdown_rx.await; - }, - ) - .await - }); + let collector = OtlpTestServer::start().await; - let (provider, error) = super::provider_for(Some(&format!("http://{address}"))); + let (provider, error) = super::provider_for(Some(collector.endpoint())); assert!(error.is_none(), "valid OTLP endpoint should configure"); let provider = provider.expect("provider"); let subscriber = tracing_subscriber::registry().with(super::layer(&provider)); @@ -242,23 +172,11 @@ mod tests { drop(span.enter()); drop(span); }); - let export_completed = exported.notified(); provider.force_flush().unwrap(); - tokio::time::timeout(std::time::Duration::from_secs(5), export_completed) - .await - .expect("OTLP export should complete"); + collector.wait_for_export().await; provider.shutdown().unwrap(); + let received = collector.shutdown().await; - shutdown_tx - .send(()) - .expect("collector server should be running"); - tokio::time::timeout(std::time::Duration::from_secs(5), server) - .await - .expect("collector server shutdown should not deadlock") - .expect("collector server task should not panic") - .expect("collector server should shut down cleanly"); - - let received = received.lock().unwrap(); received .spans .iter() diff --git a/crates/openshell-otel-test-support/Cargo.toml b/crates/openshell-otel-test-support/Cargo.toml new file mode 100644 index 0000000000..dac6ecbe7b --- /dev/null +++ b/crates/openshell-otel-test-support/Cargo.toml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-otel-test-support" +description = "Shared OTLP collector fixture for OpenShell tracing tests" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "trace"] } +tokio = { workspace = true } +tokio-stream = { workspace = true } +tonic = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-otel-test-support/src/lib.rs b/crates/openshell-otel-test-support/src/lib.rs new file mode 100644 index 0000000000..3f40d4b74a --- /dev/null +++ b/crates/openshell-otel-test-support/src/lib.rs @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared OTLP collector fixture for `OpenShell` tracing tests. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use opentelemetry_proto::tonic::collector::trace::v1::{ + ExportTraceServiceRequest, ExportTraceServiceResponse, + trace_service_server::{TraceService, TraceServiceServer}, +}; +use opentelemetry_proto::tonic::trace::v1::Span; + +#[derive(Clone, Debug, Default)] +pub struct ReceivedTraces { + pub spans: Vec, + pub service_names: Vec, +} + +#[derive(Clone)] +struct Collector { + received: Arc>, + exported: Arc, +} + +#[tonic::async_trait] +impl TraceService for Collector { + async fn export( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + { + let mut received = self + .received + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for resource_span in request.into_inner().resource_spans { + if let Some(resource) = resource_span.resource { + received.service_names.extend( + resource + .attributes + .into_iter() + .filter(|attribute| attribute.key == "service.name") + .filter_map(|attribute| attribute.value) + .filter_map(|value| value.value) + .filter_map(|value| match value { + opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => Some(value), + _ => None, + }), + ); + } + for scope_span in resource_span.scope_spans { + received.spans.extend(scope_span.spans); + } + } + } + self.exported.notify_one(); + Ok(tonic::Response::new(ExportTraceServiceResponse::default())) + } +} + +/// Loopback OTLP/gRPC server that captures exported spans and resources. +pub struct OtlpTestServer { + endpoint: String, + received: Arc>, + exported: Arc, + shutdown: tokio::sync::oneshot::Sender<()>, + task: tokio::task::JoinHandle>, +} + +impl OtlpTestServer { + pub async fn start() -> Self { + let received = Arc::new(Mutex::new(ReceivedTraces::default())); + let exported = Arc::new(tokio::sync::Notify::new()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("OTLP test collector should bind a loopback listener"); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let collector = Collector { + received: Arc::clone(&received), + exported: Arc::clone(&exported), + }; + let (shutdown, shutdown_rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(TraceServiceServer::new(collector)) + .serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + async { + let _ = shutdown_rx.await; + }, + ) + .await + }); + Self { + endpoint, + received, + exported, + shutdown, + task, + } + } + + #[must_use] + pub fn endpoint(&self) -> &str { + &self.endpoint + } + + pub async fn wait_for_export(&self) { + tokio::time::timeout(Duration::from_secs(5), self.exported.notified()) + .await + .expect("OTLP export should complete"); + } + + pub async fn shutdown(self) -> ReceivedTraces { + self.shutdown + .send(()) + .expect("OTLP test collector should still be running"); + tokio::time::timeout(Duration::from_secs(5), self.task) + .await + .expect("OTLP test collector shutdown should not deadlock") + .expect("OTLP test collector task should not panic") + .expect("OTLP test collector should shut down cleanly"); + self.received + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } +} diff --git a/crates/openshell-otel/src/lib.rs b/crates/openshell-otel/src/lib.rs index 3865670b0b..81d9f7bf01 100644 --- a/crates/openshell-otel/src/lib.rs +++ b/crates/openshell-otel/src/lib.rs @@ -199,15 +199,18 @@ pub type TargetOtlpLayer = #[derive(Debug, Clone, Copy)] pub struct TargetPrefixFilter { - prefix: &'static str, + prefix: Option<&'static str>, include: bool, } impl Filter for TargetPrefixFilter { fn enabled(&self, metadata: &tracing::Metadata<'_>, _ctx: &Context<'_, S>) -> bool { + let matches_prefix = self + .prefix + .is_some_and(|prefix| metadata.target().starts_with(prefix)); metadata.is_span() && !metadata.target().starts_with("opentelemetry") - && (metadata.target().starts_with(self.prefix) == self.include) + && (matches_prefix == self.include) } } @@ -235,16 +238,18 @@ where tracing_opentelemetry::layer() .with_tracer(provider.tracer(instrumentation_scope)) .with_filter(TargetPrefixFilter { - prefix: target_prefix, + prefix: Some(target_prefix), include: true, }) } /// Build a tracing layer that excludes spans from `target_prefix`. +/// +/// `None` excludes no application spans. pub fn layer_excluding_target_prefix( provider: &SdkTracerProvider, instrumentation_scope: &'static str, - target_prefix: &'static str, + target_prefix: Option<&'static str>, ) -> TargetOtlpLayer where S: Subscriber + for<'span> LookupSpan<'span>, @@ -332,7 +337,7 @@ mod tests { .with(layer_excluding_target_prefix( &gateway_provider, "gateway", - "driver_target", + Some("driver_target"), )) .with(layer_for_target_prefix( &driver_provider, @@ -374,6 +379,30 @@ mod tests { ); } + #[test] + fn excluding_no_target_prefix_exports_all_application_spans() { + use tracing_subscriber::layer::SubscriberExt as _; + + let _tracing_lock = test_lock(); + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry() + .with(layer_excluding_target_prefix(&provider, "gateway", None)); + + tracing::subscriber::with_default(subscriber, || { + drop(tracing::info_span!(target: "\0driver_target", "nul-prefixed.span")); + drop(tracing::info_span!(target: "gateway_target", "gateway.span")); + }); + + provider.force_flush().unwrap(); + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 2); + assert!(spans.iter().any(|span| span.name == "nul-prefixed.span")); + assert!(spans.iter().any(|span| span.name == "gateway.span")); + } + #[test] fn resource_uses_fixed_service_identity_and_custom_attributes() { let resource = resource_for(&OtlpTraceConfig { diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 4a1df2a63b..d014623dce 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -500,7 +500,7 @@ async fn run_from_args( .unwrap_or_else(|_| EnvFilter::new(&prepared.config.log_level)), &tracing_log_bus, otlp_config, - crate::tracing_setup::podman_export_enabled(&compute_driver), + &compute_driver, ); let has_client_ca = prepared diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 30a1303bd5..7ca5f339d5 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -50,7 +50,7 @@ use openshell_core::proto::{ }; use openshell_core::{ObjectLabels, ObjectWorkspace}; #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -use openshell_driver_docker::DockerComputeDriver; +use openshell_driver_docker::{ComputeDriverService as DockerDriverService, DockerComputeDriver}; #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] use openshell_driver_kubernetes::{ ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, @@ -724,11 +724,10 @@ impl ComputeRuntime { tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, ) -> Result { - let driver: SharedComputeDriver = Arc::new( - DockerComputeDriver::new(&config, &docker_config) - .await - .map_err(|err| ComputeError::Message(err.to_string()))?, - ); + let driver = DockerComputeDriver::new(&config, &docker_config) + .await + .map_err(|err| ComputeError::Message(err.to_string()))?; + let driver: SharedComputeDriver = Arc::new(DockerDriverService::new_in_process(driver)); Self::from_driver( ComputeDriverKind::Docker.as_str().to_string(), driver, diff --git a/crates/openshell-server/src/otel_tracing.rs b/crates/openshell-server/src/otel_tracing.rs index d8fb17a624..6a88b2f564 100644 --- a/crates/openshell-server/src/otel_tracing.rs +++ b/crates/openshell-server/src/otel_tracing.rs @@ -29,12 +29,6 @@ use opentelemetry_sdk::trace::SdkTracerProvider; use tracing::Subscriber; use tracing_subscriber::registry::LookupSpan; -#[cfg(feature = "in-tree-compute-drivers")] -const COMPUTE_DRIVER_TARGET_PREFIX: &str = - openshell_driver_podman::otel_tracing::IN_PROCESS_TARGET_PREFIX; -#[cfg(not(feature = "in-tree-compute-drivers"))] -const COMPUTE_DRIVER_TARGET_PREFIX: &str = "\0"; - use crate::config_file::OtlpConfig; /// `service.name` reported when the config file does not override it. @@ -92,18 +86,19 @@ pub fn provider_for(cfg: Option<&OtlpConfig>) -> (Option, Opt openshell_otel::provider_for(cfg.map(trace_config)) } -/// Build the `tracing` layer that forwards spans to `provider`. -/// -/// Events stay on the gateway's logging layers. Spans emitted by the -/// OpenTelemetry crates are excluded to prevent recursive export traffic. -pub fn layer(provider: &SdkTracerProvider) -> openshell_otel::TargetOtlpLayer +/// Build the gateway layer while routing one selected in-process driver to +/// its own tracer provider. +pub fn layer_excluding_driver( + provider: &SdkTracerProvider, + driver_target_prefix: Option<&'static str>, +) -> openshell_otel::TargetOtlpLayer where S: Subscriber + for<'span> LookupSpan<'span>, { openshell_otel::layer_excluding_target_prefix( provider, INSTRUMENTATION_SCOPE, - COMPUTE_DRIVER_TARGET_PREFIX, + driver_target_prefix, ) } @@ -137,7 +132,8 @@ pub mod test_exporter { let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() .with_simple_exporter(exporter.clone()) .build(); - let subscriber = tracing_subscriber::registry().with(super::layer(&provider)); + let subscriber = + tracing_subscriber::registry().with(super::layer_excluding_driver(&provider, None)); let dispatch = tracing::Dispatch::new(subscriber); TracingTestGuard { _default: tracing::dispatcher::set_default(&dispatch), diff --git a/crates/openshell-server/src/tracing_setup.rs b/crates/openshell-server/src/tracing_setup.rs index aff5c05ecf..bbd98f16c6 100644 --- a/crates/openshell-server/src/tracing_setup.rs +++ b/crates/openshell-server/src/tracing_setup.rs @@ -18,7 +18,7 @@ use crate::tracing_bus::TracingLogBus; pub struct TracingHandle { tracer_provider: Option, - podman_tracer_provider: Option, + driver_tracer_provider: Option, } impl TracingHandle { @@ -28,78 +28,109 @@ impl TracingHandle { { tracing::warn!(error = %err, "OTLP tracer provider shutdown failed"); } - if let Some(provider) = &self.podman_tracer_provider + if let Some(provider) = &self.driver_tracer_provider && let Err(err) = provider.shutdown() { - tracing::warn!(error = %err, "Podman OTLP tracer provider shutdown failed"); + tracing::warn!(error = %err, "compute-driver OTLP tracer provider shutdown failed"); } } } -#[must_use] -pub fn podman_export_enabled(driver: &ConfiguredComputeDriver) -> bool { - matches!( - driver, - ConfiguredComputeDriver::Registered(registration) if registration.name == "podman" - ) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum InProcessDriverTracing { + #[cfg(feature = "in-tree-compute-drivers")] + Docker, + #[cfg(feature = "in-tree-compute-drivers")] + Podman, } -pub fn install( - env_filter: EnvFilter, - tracing_log_bus: &TracingLogBus, - otlp_config: Option<&OtlpConfig>, - enable_podman_export: bool, -) -> (TracingHandle, Option) { - let (tracer_provider, setup_error) = crate::otel_tracing::provider_for(otlp_config); +impl InProcessDriverTracing { #[cfg(feature = "in-tree-compute-drivers")] - let podman_endpoint = enable_podman_export - .then_some(otlp_config) - .flatten() - .map(|config| config.endpoint.as_str()); + fn target_prefix(self) -> &'static str { + match self { + Self::Docker => openshell_driver_docker::otel_tracing::IN_PROCESS_TARGET_PREFIX, + Self::Podman => openshell_driver_podman::otel_tracing::IN_PROCESS_TARGET_PREFIX, + } + } +} + +fn in_process_driver_tracing(driver: &ConfiguredComputeDriver) -> Option { #[cfg(feature = "in-tree-compute-drivers")] - let (podman_tracer_provider, podman_setup_error) = - openshell_driver_podman::otel_tracing::provider_for(podman_endpoint); + match driver { + ConfiguredComputeDriver::Registered(registration) if registration.name == "docker" => { + Some(InProcessDriverTracing::Docker) + } + ConfiguredComputeDriver::Registered(registration) if registration.name == "podman" => { + Some(InProcessDriverTracing::Podman) + } + _ => None, + } #[cfg(not(feature = "in-tree-compute-drivers"))] - let (podman_tracer_provider, podman_setup_error): ( - Option, - Option, - ) = { - let _ = enable_podman_export; - (None, None) - }; + { + let _ = driver; + None + } +} - tracing_subscriber::registry() - .with(env_filter) - .with(tracing_subscriber::fmt::layer()) - .with(tracing_log_bus.layer()) - .with(tracer_provider.as_ref().map(crate::otel_tracing::layer)) - .with(podman_in_process_layer(&podman_tracer_provider)) - .init(); +fn in_process_driver_target_prefix(driver: Option) -> Option<&'static str> { + #[cfg(feature = "in-tree-compute-drivers")] + { + driver.map(InProcessDriverTracing::target_prefix) + } + #[cfg(not(feature = "in-tree-compute-drivers"))] + { + let _ = driver; + None + } +} - ( - TracingHandle { - tracer_provider, - podman_tracer_provider, - }, - setup_error.or(podman_setup_error), - ) +#[cfg(feature = "in-tree-compute-drivers")] +fn in_process_driver_provider( + driver: Option, + endpoint: Option<&str>, +) -> (Option, Option) { + match driver { + Some(InProcessDriverTracing::Docker) => { + openshell_driver_docker::otel_tracing::provider_for(endpoint) + } + Some(InProcessDriverTracing::Podman) => { + openshell_driver_podman::otel_tracing::provider_for(endpoint) + } + None => (None, None), + } +} + +#[cfg(not(feature = "in-tree-compute-drivers"))] +fn in_process_driver_provider( + _driver: Option, + _endpoint: Option<&str>, +) -> (Option, Option) { + (None, None) } #[cfg(feature = "in-tree-compute-drivers")] -fn podman_in_process_layer( +fn in_process_driver_layer( provider: &Option, + driver: Option, ) -> Option> where S: tracing::Subscriber + for<'span> tracing_subscriber::registry::LookupSpan<'span>, { - provider - .as_ref() - .map(openshell_driver_podman::otel_tracing::in_process_layer) + provider.as_ref().map(|provider| match driver { + Some(InProcessDriverTracing::Docker) => { + openshell_driver_docker::otel_tracing::in_process_layer(provider) + } + Some(InProcessDriverTracing::Podman) => { + openshell_driver_podman::otel_tracing::in_process_layer(provider) + } + None => unreachable!("a driver provider requires a selected driver"), + }) } #[cfg(not(feature = "in-tree-compute-drivers"))] -fn podman_in_process_layer( +fn in_process_driver_layer( _provider: &Option, + _driver: Option, ) -> Option> where S: tracing::Subscriber + for<'span> tracing_subscriber::registry::LookupSpan<'span>, @@ -107,13 +138,54 @@ where None } +pub fn install( + env_filter: EnvFilter, + tracing_log_bus: &TracingLogBus, + otlp_config: Option<&OtlpConfig>, + driver: &ConfiguredComputeDriver, +) -> (TracingHandle, Option) { + let (tracer_provider, setup_error) = crate::otel_tracing::provider_for(otlp_config); + let selected_driver = in_process_driver_tracing(driver); + let driver_endpoint = selected_driver + .is_some() + .then_some(otlp_config) + .flatten() + .map(|config| config.endpoint.as_str()); + let (driver_tracer_provider, driver_setup_error) = + in_process_driver_provider(selected_driver, driver_endpoint); + + tracing_subscriber::registry() + .with(env_filter) + .with(tracing_subscriber::fmt::layer()) + .with(tracing_log_bus.layer()) + .with(tracer_provider.as_ref().map(|provider| { + crate::otel_tracing::layer_excluding_driver( + provider, + in_process_driver_target_prefix(selected_driver), + ) + })) + .with(in_process_driver_layer( + &driver_tracer_provider, + selected_driver, + )) + .init(); + + ( + TracingHandle { + tracer_provider, + driver_tracer_provider, + }, + setup_error.or(driver_setup_error), + ) +} + #[cfg(test)] mod tests { use super::*; - #[cfg(not(target_os = "windows"))] + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[test] - fn podman_export_is_enabled_only_when_podman_is_selected() { + fn in_process_driver_tracing_selects_docker_and_podman() { let registry = crate::install_default_compute_drivers(); let registered = |name| { ConfiguredComputeDriver::Registered( @@ -123,12 +195,20 @@ mod tests { .clone(), ) }; - - assert!(podman_export_enabled(®istered("podman"))); - assert!(!podman_export_enabled(®istered("docker"))); - assert!(!podman_export_enabled(®istered("kubernetes"))); - assert!(!podman_export_enabled(&ConfiguredComputeDriver::Remote { - name: "custom".to_string(), - })); + assert_eq!( + in_process_driver_tracing(®istered("podman")), + Some(InProcessDriverTracing::Podman) + ); + assert_eq!( + in_process_driver_tracing(®istered("docker")), + Some(InProcessDriverTracing::Docker) + ); + assert_eq!(in_process_driver_tracing(®istered("kubernetes")), None); + assert_eq!( + in_process_driver_tracing(&ConfiguredComputeDriver::Remote { + name: "custom".to_string(), + }), + None + ); } } diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index cefae0b5cb..83b312bcc9 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -232,7 +232,7 @@ The OpenTelemetry SDK logs export failures after startup. Spans in a failed batc Only OpenTelemetry traces are exported. Inbound gRPC and HTTP requests produce server spans named for the RPC or HTTP method. Store and compute-driver operations appear as child spans. Internal reconciliation, credential-refresh, and driver-watch loops create operation roots for their store work because no inbound request supplies a parent. The gateway continues valid W3C `traceparent` context and starts a new trace when none is supplied. Request spans carry `method`, `path`, and the `request_id` that also appears in gateway logs. Health endpoint spans use DEBUG level and are not exported by the default INFO filter. -The gateway forwards the OTLP configuration to managed external drivers. Each driver exports under its own service name. +The gateway forwards the OTLP configuration and W3C trace context to managed external drivers. Built-in Docker and Podman drivers also export their spans to the same collector through dedicated in-process providers. Driver spans retain the gateway trace context and use a distinct service name such as `openshell-driver-docker` or `openshell-driver-podman`. ### Tuning