Local LGTM observability stack for engine telemetry

CHANGES

@@ -277,6 +277,38 @@ it.
 
 ### Development
 
+#### Local observability stack
+
+`just otel-verify` starts a Grafana LGTM container, drives a real tmux workload
+through every engine transport, and checks that each dashboard panel returns
+data. Metrics, traces, logs, and continuous profiles are emitted through the
+engine instrumentation seam, so the exporters are ordinary sinks and libtmux
+itself gains no OpenTelemetry dependency.
+
+The Grafana dashboards under `scripts/lgtm/dashboards/` are generated rather
+than hand-edited, and `scripts/lgtm/acceptance.py` runs each panel's own query
+and fails naming any panel that came back empty. See `scripts/lgtm/README.md`.
+
+Every run is stamped with its branch, revision, repository, worktree, and an
+optional spike name, so two runs can be compared rather than merely observed.
+Which signal carries which fact is a deliberate split: metrics take only the
+dimensions worth grouping by, while the revision, worktree, and per-test
+identity ride on traces and profiles where high cardinality is expected.
+Values that change within a process, such as the phase a workload is in, travel
+as OpenTelemetry baggage and are copied onto spans by a processor instead of
+being threaded through engine calls.
+
+`just otel-load` drives the engines under a ramping arrival rate, an open model
+that keeps offering work regardless of latency and so exposes where a transport
+saturates -- something a fixed-worker loop hides, because it slows down along
+with the system. The schedule comes from rampa; the telemetry still comes from
+this project's own sink, so a load-shaped run is directly comparable with every
+other run rather than speaking a second vocabulary.
+
+The dashboards follow Grafana's guidance: a Home board says which board answers
+which question, Overview is organized as rate, errors, and duration, and its
+panels link down to the board that explains them.
+
 #### CI actions updated to current majors
 
 Workflow actions moved to their current major releases: `actions/checkout` v7,

docs/experimental/instrumentation.md

@@ -191,6 +191,11 @@ sink exports over OTLP, where `{ span.tmux.commands > 1 }` finds every request
 that batched work. Nothing else changes, because the sink never learns which
 engine it is observing.
 
+That exporter already exists, along with a local Grafana stack to receive it and
+dashboards built on these counts. `just otel-verify` starts it, drives a tmux
+workload through the seam, and checks that every dashboard panel has data. See
+`scripts/lgtm/README.md`.
+
 ## Async
 
 {func}`~libtmux.experimental.engines.instrumentation.instrument` returns
@@ -282,9 +287,10 @@ hooks would build a context object per call whether or not one is used. Wrapping
 adds one Python call plus each sink's own work, and only for programs that ask
 for it.
 
-Two tests hold the claim: one asserts wrapping adds no attribute to the engine
-it wraps, and one measures an instrumented call against the millisecond scale of
-a real tmux round trip.
+The claim is held by two tests: one asserts that wrapping adds no attribute to
+the engine it wraps, and one measures an instrumented call against the
+millisecond scale of a real tmux round trip.
+
 ```console
 $ uv run pytest tests/experimental/engines/test_instrumentation.py
 ```

justfile

@@ -142,3 +142,50 @@ _entr-warn:
     @echo "Install entr(1) to automatically run tasks on file change."
     @echo "See https://eradman.com/entrproject/                      "
     @echo "----------------------------------------------------------"
+
+# ---- OpenTelemetry / LGTM dev workflow ----
+
+# Start the local Grafana LGTM stack the telemetry checks query against
+[group: 'otel']
+otel-up:
+    scripts/lgtm/up.sh
+
+# Stop and remove the local Grafana LGTM stack
+[group: 'otel']
+otel-down:
+    docker rm -f ${LIBTMUX_LGTM_CONTAINER:-libtmux-lgtm}
+
+# Confirm each published port reaches the container, not a host process
+[group: 'otel']
+otel-ports:
+    scripts/lgtm/verify.sh
+
+# Print an MCP client config for this stack, with its real ports and token
+[group: 'otel']
+otel-mcp:
+    scripts/lgtm/mcp-config.sh
+
+# Regenerate the provisioned Grafana dashboards from their generator
+[group: 'otel']
+otel-dashboards:
+    uv run python scripts/lgtm/generate_dashboards.py
+
+# Drive a real tmux workload through the engine seam into LGTM
+[group: 'otel']
+otel-smoke *args:
+    uv run --group otel python scripts/lgtm/smoke.py {{ args }}
+
+# Verify every dashboard panel's own queries return data
+[group: 'otel']
+otel-acceptance *args:
+    uv run --group otel python scripts/lgtm/acceptance.py {{ args }}
+
+# Start the stack, run the workload, then verify every panel end to end
+[group: 'otel']
+otel-verify:
+    uv run --group otel python scripts/lgtm/acceptance.py --start-stack --smoke
+
+# Drive the engines under a load shape (ramping arrival rate) via rampa
+[group: 'otel']
+otel-load *args:
+    uv run --group otel --group load rampa run scripts/lgtm/load_tmux.py {{ args }}

pyproject.toml

@@ -53,6 +53,20 @@ Repository = "https://github.com/tmux-python/libtmux"
 Changes = "https://github.com/tmux-python/libtmux/blob/master/CHANGES"
 
 [dependency-groups]
+# Telemetry exporters for the local LGTM stack (scripts/lgtm/, just otel-*).
+# Kept out of `dev` so the ordinary test and type gates stay lean: libtmux
+# itself never imports OpenTelemetry, and only scripts/ do.
+otel = [
+  "opentelemetry-sdk",
+  "opentelemetry-exporter-otlp-proto-http",
+  "pyroscope-io",
+]
+# Load shaping for scripts/lgtm/load_tmux.py (just otel-load). Separate from
+# `otel` because rampa needs a newer Python than libtmux supports, so the
+# marker keeps `uv sync` working on every version libtmux targets.
+load = [
+  "rampa; python_version >= '3.14'",
+]
 dev = [
   # Docs (via gp-sphinx)
   "gp-sphinx==0.1.0a37",
@@ -172,6 +186,30 @@ files = [
   "tests",
 ]
 
+# The observability scripts import third-party packages from the optional
+# `otel` group, which CI does not install so the ordinary gates stay lean.
+# `mypy .` still reaches those scripts, so their imports must be allowed to
+# be absent. `rampa` ships no stubs either way.
+[[tool.mypy.overrides]]
+module = [
+  "opentelemetry.*",
+  "pyroscope",
+  "rampa",
+]
+ignore_missing_imports = true
+
+# Those same packages ship no stubs, so anything derived from them is `Any`:
+# a SpanProcessor subclass and rampa's scenario decorators. Relaxing the two
+# strictness flags here is scoped to these scripts and, unlike an inline
+# ignore, stays correct whether or not the optional group is installed.
+[[tool.mypy.overrides]]
+module = [
+  "telemetry",
+  "load_tmux",
+]
+disallow_subclassing_any = false
+disallow_untyped_decorators = false
+
 
 [tool.ty.environment]
 python-version = "3.10"