From e00b721256a9597e514b7c411e3a1ec0efa7f064 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Fri, 21 Aug 2026 13:19:16 -0500 Subject: [PATCH 01/24] Bench(fix[stress]): Scale the progress watchdog with the rung why: Every rung inherited the benchmark's flat 120s no-progress watchdog, whatever topology it built. The cost per pass is superlinear on this ladder's own shapes -- one control/async pass measures 14.3s at 400 panes, 24.0s at 800, 52.5s at 1200 and 177.5s at 1600 -- so the allowance that is generous at the base is short at the top, and a slower rung is reported as a stuck one. That stops the axis before it reaches any real limit, which is the opposite of what an escalating harness is for. what: - Give each rung an allowance derived from the panes it builds, and pass it through to the benchmark - Cap it at the hard per-rung limit, so a genuinely stuck rung dies no later than it would have - Cover the scaling, the cap, and that the value actually reaches the child --- scripts/orchestration/stress.py | 47 +++++++++++++++ tests/scripts/orchestration/test_stress.py | 68 ++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/scripts/orchestration/stress.py b/scripts/orchestration/stress.py index f9bc5df15..772ccce0c 100755 --- a/scripts/orchestration/stress.py +++ b/scripts/orchestration/stress.py @@ -51,6 +51,14 @@ _BENCHMARK = pathlib.Path(__file__).with_name("benchmark.py") _SCRATCH_PREFIX = "lts-" _MEMORY_FLOOR_BYTES = 8 * 1024**3 +# The benchmark's own progress watchdog is a flat 120s, which does not scale +# with the topology a rung builds. Measured on this ladder's own shapes, one +# control/async pass costs 14.3s at 400 panes, 24.0s at 800, 52.5s at 1200 and +# 177.5s at 1600 -- superlinear, so a fixed allowance turns into a false +# failure exactly where the harness is meant to be measuring. These give a +# per-rung allowance instead, still bounded by --rung-timeout-seconds. +_WATCHDOG_BASE_S = 120.0 +_WATCHDOG_PER_PANE_S = 0.25 @functools.lru_cache(maxsize=1) @@ -251,6 +259,43 @@ def rung_outcome(artifact: pathlib.Path, returncode: int | None) -> dict[str, ob } +def watchdog_seconds(shape: Shape, rung_timeout_s: float) -> float: + """Return the progress-gap allowance for *shape*, capped by the rung limit. + + The allowance has to grow with the topology: the same phase that finishes + well inside 120 seconds at 400 panes needs several times that at 1600, so a + flat watchdog reports a slow rung as a stuck one and stops the ladder short + of anything real. The cap keeps a genuinely stuck rung from outliving the + hard per-rung limit that would have killed it anyway. + + Parameters + ---------- + shape : Shape + Topology this rung builds. + rung_timeout_s : float + Hard limit for the whole rung, which the allowance never exceeds. + + Returns + ------- + float + Seconds a rung may make no progress before it is declared stuck. + + Examples + -------- + >>> watchdog_seconds(Shape(80, 20, 1), 900.0) + 520.0 + >>> watchdog_seconds(Shape(20, 20, 1), 900.0) + 220.0 + + A large topology is bounded by the rung limit rather than the formula: + + >>> watchdog_seconds(Shape(80, 20, 4), 900.0) + 900.0 + """ + scaled = _WATCHDOG_BASE_S + shape.panes * _WATCHDOG_PER_PANE_S + return min(scaled, rung_timeout_s) + + def run_rung( shape: Shape, *, @@ -295,6 +340,8 @@ def run_rung( str(out / "report.json"), "--scratch-root", str(scratch), + "--watchdog-seconds", + str(watchdog_seconds(shape, timeout_s)), ) started = time.monotonic() returncode: int | None = None diff --git a/tests/scripts/orchestration/test_stress.py b/tests/scripts/orchestration/test_stress.py index 1e84cb19a..be5d66f6f 100644 --- a/tests/scripts/orchestration/test_stress.py +++ b/tests/scripts/orchestration/test_stress.py @@ -64,6 +64,74 @@ def test_ladders_escalate_pane_pressure(stress_module: types.ModuleType) -> None assert panes[-1] > panes[0] +def test_watchdog_allowance_grows_with_the_rung( + stress_module: types.ModuleType, +) -> None: + """Every rung on a ladder gets at least as much slack as the one below it. + + The benchmark's own default is flat, and one control/async pass measured + 14.3s at 400 panes against 177.5s at 1600. A fixed allowance therefore + reports the larger rung as stuck when it is only slower, which stops the + ladder before it reaches anything real. + """ + for axis in ("panes", "windows", "sessions"): + allowances = [ + stress_module.watchdog_seconds(rung, 3600.0) + for rung in stress_module.ladder(axis) + ] + assert allowances == sorted(allowances) + assert allowances[-1] > allowances[0] + assert min(allowances) > 120.0, "every rung beats the flat default" + + +def test_watchdog_allowance_never_outlives_the_rung_limit( + stress_module: types.ModuleType, +) -> None: + """A stuck rung is still killed by the hard per-rung limit.""" + biggest = max(stress_module.ladder("panes"), key=lambda rung: rung.panes) + + assert stress_module.watchdog_seconds(biggest, 90.0) == 90.0 + + +def test_rung_passes_its_scaled_watchdog_to_the_benchmark( + stress_module: types.ModuleType, + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, +) -> None: + """The allowance has to reach the child, not just be computable.""" + shape = stress_module.Shape(80, 20, 1) + seen: list[tuple[str, ...]] = [] + + class _Finished: + returncode = 0 + + def wait(self, timeout: float | None = None) -> int: + return 0 + + def record(command: tuple[str, ...], **kwargs: object) -> _Finished: + seen.append(tuple(command)) + return _Finished() + + # Resolved rather than probed: child_interpreter() shells out itself, and + # a Popen stub would otherwise intercept that probe instead of the rung. + monkeypatch.setattr(stress_module, "child_interpreter", lambda: sys.executable) + monkeypatch.setattr(stress_module.subprocess, "Popen", record) + stress_module.run_rung( + shape, + lane="control", + mode="async", + evidence_root=tmp_path, + timeout_s=900.0, + ) + + assert seen, "the benchmark was never invoked" + command = seen[0] + assert "--watchdog-seconds" in command + passed = float(command[command.index("--watchdog-seconds") + 1]) + assert passed == stress_module.watchdog_seconds(shape, 900.0) + assert passed > 120.0 + + def test_shape_supports_column_alignment(stress_module: types.ModuleType) -> None: """The reporting loop aligns shapes, which a bare dataclass rejects.""" shape = stress_module.Shape(80, 20, 2) From 382efc6ac75dd1648a47673459c6f41c6e15403d Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 08:15:05 -0500 Subject: [PATCH 02/24] Bench(feat[lgtm]): Add the local observability stack why: The engine instrumentation seam could emit telemetry but had nowhere to send it, so nobody could see what a tmux workload actually costs. Standing up that stack by hand is unreproducible, and a dashboard nobody queries rots into panels that render an empty grid and look healthy doing it. The workload is short-lived, so every panel has to read the window the viewer selected rather than a counter at an instant: Prometheus marks a finished run's series stale within minutes, and an instant query then returns nothing beside a timeseries still drawing the same run. what: - Add scripts/lgtm/up.sh, pinning the otel-lgtm image and bind-mounting the datasource and dashboard provisioning, with a config label so changing a mount recreates the container instead of restarting it with stale state - Publish Grafana on 3900 and Prometheus on 9099: a host process already bound to their defaults still lets Docker publish, then answers first, so queries reach the wrong server and return plausible data - Add scripts/lgtm/telemetry.py, an OTelSink emitting spans and metrics together so the duration histogram records while its span is current and carries an exemplar, giving the metric-to-trace pivot - Add scripts/otel_smoke.py driving all four transports, with grouped commands so the inlining panels are not flat zero and rejected commands so the failure panels have real data - Generate the three dashboards from scripts/lgtm/generate_dashboards.py rather than hand-editing JSON, and commit the output - Query each panel's own window with increase(...[$__range]) rather than reading counters at a point, so a panel reports what happened in the range the viewer selected - Add scripts/otel_acceptance.py, which reads those dashboards, runs every panel's own query the way Grafana runs it -- range or instant -- and fails naming any that returned nothing; it re-checks until ingestion catches up, since "no data yet" and "no data ever" are indistinguishable at any single instant - Reject any Prometheus target with no range selector in a test, which catches a stale-series panel offline; a live check only sees it once the series has gone stale, which is precisely when nobody is watching - Add just otel-up/down/dashboards/smoke/acceptance/verify and an otel dependency group, kept out of dev so the ordinary gates stay lean --- CHANGES | 12 + justfile | 32 + pyproject.toml | 8 + scripts/lgtm/README.md | 145 +++ scripts/lgtm/dashboards/libtmux-commands.json | 417 +++++++++ scripts/lgtm/dashboards/libtmux-overview.json | 659 ++++++++++++++ .../lgtm/dashboards/libtmux-transports.json | 543 +++++++++++ scripts/lgtm/generate_dashboards.py | 843 ++++++++++++++++++ scripts/lgtm/grafana-dashboards-libtmux.yaml | 16 + scripts/lgtm/grafana-datasources.yaml | 66 ++ scripts/lgtm/telemetry.py | 238 +++++ scripts/lgtm/up.sh | 97 ++ scripts/otel_acceptance.py | 353 ++++++++ scripts/otel_smoke.py | 275 ++++++ tests/test_lgtm_dashboards.py | 179 ++++ uv.lock | 150 +++- 16 files changed, 4030 insertions(+), 3 deletions(-) create mode 100644 scripts/lgtm/README.md create mode 100644 scripts/lgtm/dashboards/libtmux-commands.json create mode 100644 scripts/lgtm/dashboards/libtmux-overview.json create mode 100644 scripts/lgtm/dashboards/libtmux-transports.json create mode 100644 scripts/lgtm/generate_dashboards.py create mode 100644 scripts/lgtm/grafana-dashboards-libtmux.yaml create mode 100644 scripts/lgtm/grafana-datasources.yaml create mode 100644 scripts/lgtm/telemetry.py create mode 100755 scripts/lgtm/up.sh create mode 100644 scripts/otel_acceptance.py create mode 100644 scripts/otel_smoke.py create mode 100644 tests/test_lgtm_dashboards.py diff --git a/CHANGES b/CHANGES index 977b3921b..26125b4a9 100644 --- a/CHANGES +++ b/CHANGES @@ -277,6 +277,18 @@ it. ### Development +#### Local observability stack + +`just otel-verify` starts a Grafana LGTM container, drives a real tmux workload +through every engine transport, and checks that each dashboard panel returns +data. Metrics, traces, logs, and continuous profiles are emitted through the +engine instrumentation seam, so the exporters are ordinary sinks and libtmux +itself gains no OpenTelemetry dependency. + +The Grafana dashboards under `scripts/lgtm/dashboards/` are generated rather +than hand-edited, and `scripts/otel_acceptance.py` runs each panel's own query +and fails naming any panel that came back empty. See `scripts/lgtm/README.md`. + #### CI actions updated to current majors Workflow actions moved to their current major releases: `actions/checkout` v7, diff --git a/justfile b/justfile index b8e59fba1..57e4f5efc 100644 --- a/justfile +++ b/justfile @@ -142,3 +142,35 @@ _entr-warn: @echo "Install entr(1) to automatically run tasks on file change." @echo "See https://eradman.com/entrproject/ " @echo "----------------------------------------------------------" + +# ---- OpenTelemetry / LGTM dev workflow ---- + +# Start the local Grafana LGTM stack the telemetry checks query against +[group: 'otel'] +otel-up: + scripts/lgtm/up.sh + +# Stop and remove the local Grafana LGTM stack +[group: 'otel'] +otel-down: + docker rm -f ${LIBTMUX_LGTM_CONTAINER:-libtmux-lgtm} + +# Regenerate the provisioned Grafana dashboards from their generator +[group: 'otel'] +otel-dashboards: + uv run python scripts/lgtm/generate_dashboards.py + +# Drive a real tmux workload through the engine seam into LGTM +[group: 'otel'] +otel-smoke *args: + uv run --group otel python scripts/otel_smoke.py {{ args }} + +# Verify every dashboard panel's own queries return data +[group: 'otel'] +otel-acceptance *args: + uv run --group otel python scripts/otel_acceptance.py {{ args }} + +# Start the stack, run the workload, then verify every panel end to end +[group: 'otel'] +otel-verify: + uv run --group otel python scripts/otel_acceptance.py --start-stack --smoke diff --git a/pyproject.toml b/pyproject.toml index 33921110c..e7562afcd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,14 @@ Repository = "https://github.com/tmux-python/libtmux" Changes = "https://github.com/tmux-python/libtmux/blob/master/CHANGES" [dependency-groups] +# Telemetry exporters for the local LGTM stack (scripts/lgtm/, just otel-*). +# Kept out of `dev` so the ordinary test and type gates stay lean: libtmux +# itself never imports OpenTelemetry, and only scripts/ do. +otel = [ + "opentelemetry-sdk", + "opentelemetry-exporter-otlp-proto-http", + "pyroscope-io", +] dev = [ # Docs (via gp-sphinx) "gp-sphinx==0.1.0a37", diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md new file mode 100644 index 000000000..0cc6b5ffe --- /dev/null +++ b/scripts/lgtm/README.md @@ -0,0 +1,145 @@ +# Local observability stack + +A single container running Grafana, Loki, Tempo, Prometheus, Pyroscope, and an +OpenTelemetry collector, plus the libtmux dashboards that read from it. It +exists to answer what a tmux workload actually costs: how many commands each +transport issues, how long they take, which ones tmux rejects, and where Python +spent its time getting there. + +Nothing here is imported by libtmux. Telemetry is emitted through the engine +instrumentation seam, so the exporters are ordinary sinks and the engines are +untouched. See [`docs/experimental/instrumentation.md`](../../docs/experimental/instrumentation.md) +for the seam itself. + +## Run the whole thing + +Start the stack, drive a real tmux workload through it, and verify every +dashboard panel returns data: + +```console +$ just otel-verify +``` + +That is the command to reach for first. The steps below are the same workflow +taken one piece at a time. + +Start or restart the stack: + +```console +$ just otel-up +``` + +Drive a workload: + +```console +$ just otel-smoke +``` + +Check that every panel has data: + +```console +$ just otel-acceptance +``` + +Stop it: + +```console +$ just otel-down +``` + +## Ports + +| Service | URL | | +| ------- | --- | --- | +| Grafana | | `admin` / `admin` | +| Prometheus | | metrics | +| Tempo | | traces, MCP at `/api/mcp` | +| Pyroscope | | profiles | +| Loki | | logs | +| OTLP | `4317` gRPC, `4318` HTTP | ingest | + +Grafana avoids 3000 and Prometheus avoids 9090 on purpose. Both defaults are +commonly taken on a dev box, and a taken port does not fail loudly: Docker still +publishes it, but a host process already bound there answers first. Queries then +reach the wrong server and return plausible data, which costs far more +debugging time than a refused connection. Override with +`LIBTMUX_LGTM_GRAFANA_PORT` and `LIBTMUX_LGTM_PROM_PORT` if these collide too. + +## What the workload emits + +`scripts/otel_smoke.py` runs every transport — subprocess and control mode, +sync and async — against a throwaway tmux server, and emits four signals: + +Metrics are `tmux_requests_total`, `tmux_commands_total`, `tmux_inlined_total`, +`tmux_failures_total`, and the `tmux_command_duration_seconds` histogram, each +labelled by `tmux_lane` and `tmux_command`. + +Traces are one span per command, carrying `tmux.commands` and `tmux.inlined` so +TraceQL can find requests that batched work. The duration histogram records +while its span is current, which attaches an exemplar and gives Grafana the +metric-to-trace pivot. + +Logs carry trace context, so a log line links to the trace it came from. + +Profiles come from Pyroscope sampling the process, which is how "where did the +Python time go" gets answered — the engines' own frames show up in the flame +graph. + +The workload deliberately issues commands tmux rejects. A dashboard whose error +panel is empty is untested rather than healthy, so the failure path has to +produce real data. + +## Dashboards + +Three boards, provisioned into the `libtmux` folder: + +`libtmux / Overview` is throughput, latency, failures, and the trace, log, and +profile panels side by side. `libtmux / Transports` compares the four transports +against each other. `libtmux / Commands` breaks the same work down by tmux +command. + +The JSON is generated, not hand-written, by `generate_dashboards.py`. A board is +a few hundred lines of nested objects where panel placement is manual +arithmetic, so hand-maintaining three of them guarantees drift. Regenerate after +changing the generator: + +```console +$ just otel-dashboards +``` + +`up.sh` regenerates on every start, and `tests/test_lgtm_dashboards.py` fails if +the committed JSON differs from what the generator produces, so the two cannot +diverge quietly. Editing a board in the Grafana UI is fine for exploring; move +the change into the generator to keep it. + +## Why the acceptance check exists + +A dashboard that renders is not a dashboard that works. A panel whose query +returns nothing looks exactly like a panel reporting a healthy zero. + +`scripts/otel_acceptance.py` reads the generated JSON, expands the template +variables the way Grafana would, runs every panel's own query against +Prometheus, Loki, Tempo, or Pyroscope, and fails naming any panel that returned +nothing. Because it reads the dashboards themselves, a panel added to the +generator is covered the moment it exists. + +Ingestion is asynchronous, and each backend buffers on its own schedule, so at +any single instant "no data yet" is indistinguishable from "no data ever". The +check therefore re-queries the panels that came back empty until they fill or +`--timeout` expires; a warm stack passes on the first sweep and a cold one waits +only as long as it needs. A cold-started Tempo is the usual reason for a +second pass. + +## Configuration + +`up.sh` pins the `grafana/otel-lgtm` image rather than tracking `latest`, so a +rerun sees the Prometheus and Pyroscope this stack was verified against. It +bind-mounts `grafana-datasources.yaml` (pinning the datasource uids every panel +binds to), `grafana-dashboards-libtmux.yaml` (the provider), and the generated +`dashboards/` directory. + +The container carries a config label. Changing the mounted configuration means +bumping `CONFIG_LABEL` in `up.sh`, which makes the next `just otel-up` recreate +the container instead of restarting it with stale mounts. + +Upstream image: diff --git a/scripts/lgtm/dashboards/libtmux-commands.json b/scripts/lgtm/dashboards/libtmux-commands.json new file mode 100644 index 000000000..a077662bc --- /dev/null +++ b/scripts/lgtm/dashboards/libtmux-commands.json @@ -0,0 +1,417 @@ +{ + "uid": "libtmux-commands", + "title": "libtmux / Commands", + "description": "Which tmux commands the workload issues, and what they cost.", + "tags": [ + "libtmux", + "generated" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "lane", + "label": "Transport", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(tmux_requests_total, tmux_lane)", + "refId": "var-lane" + }, + "refresh": 2, + "sort": 1, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "links": [ + { + "title": "libtmux dashboards", + "type": "dashboards", + "tags": [ + "libtmux" + ], + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "icon": "external link" + } + ], + "annotations": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "Command mix", + "collapsed": false, + "id": 1, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "piechart", + "title": "Requests by command", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_command) (increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range]))", + "legendFormat": "{{tmux_command}}", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "id": 2, + "gridPos": { + "x": 0, + "y": 1, + "w": 8, + "h": 9 + } + }, + { + "type": "timeseries", + "title": "Request rate by command", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_command) (rate(tmux_requests_total{tmux_lane=~\"$lane\"}[$__rate_interval]))", + "legendFormat": "{{tmux_command}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps", + "custom": { + "fillOpacity": 18, + "showPoints": "never", + "lineWidth": 2, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 3, + "gridPos": { + "x": 8, + "y": 1, + "w": 16, + "h": 9 + } + }, + { + "type": "row", + "title": "Cost and failures", + "collapsed": false, + "id": 4, + "gridPos": { + "x": 0, + "y": 10, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "p95 by command", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, tmux_command) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__rate_interval])))", + "legendFormat": "{{tmux_command}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "s", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 5, + "gridPos": { + "x": 0, + "y": 11, + "w": 12, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "Failures by command", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_command) (rate(tmux_failures_total{tmux_lane=~\"$lane\"}[$__rate_interval]))", + "legendFormat": "{{tmux_command}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 6, + "gridPos": { + "x": 12, + "y": 11, + "w": 12, + "h": 8 + } + }, + { + "type": "table", + "title": "Command summary", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_command) (increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range]))", + "legendFormat": "requests", + "range": false, + "instant": true, + "refId": "A", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_command) (increase(tmux_commands_total{tmux_lane=~\"$lane\"}[$__range]))", + "legendFormat": "commands", + "range": false, + "instant": true, + "refId": "B", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, tmux_command) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__range])))", + "legendFormat": "p95", + "range": false, + "instant": true, + "refId": "C", + "format": "table" + } + ], + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true, + "instance": true + } + } + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto" + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "footer": { + "show": false + } + }, + "id": 7, + "gridPos": { + "x": 0, + "y": 19, + "w": 24, + "h": 8 + } + } + ] +} diff --git a/scripts/lgtm/dashboards/libtmux-overview.json b/scripts/lgtm/dashboards/libtmux-overview.json new file mode 100644 index 000000000..5d936b27c --- /dev/null +++ b/scripts/lgtm/dashboards/libtmux-overview.json @@ -0,0 +1,659 @@ +{ + "uid": "libtmux-overview", + "title": "libtmux / Overview", + "description": "Throughput, latency, and failures across every engine transport. Metric panels carry exemplars, so a latency spike links to the trace behind it.", + "tags": [ + "libtmux", + "generated" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "lane", + "label": "Transport", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(tmux_requests_total, tmux_lane)", + "refId": "var-lane" + }, + "refresh": 2, + "sort": 1, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "links": [ + { + "title": "libtmux dashboards", + "type": "dashboards", + "tags": [ + "libtmux" + ], + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "icon": "external link" + } + ], + "annotations": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "Totals", + "collapsed": false, + "id": 1, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "stat", + "title": "Requests", + "description": "Requests dispatched to an engine.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range]))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "id": 2, + "gridPos": { + "x": 0, + "y": 1, + "w": 6, + "h": 5 + } + }, + { + "type": "stat", + "title": "tmux commands", + "description": "Commands tmux was told to run; a group counts as its members.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(tmux_commands_total{tmux_lane=~\"$lane\"}[$__range]))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "id": 3, + "gridPos": { + "x": 6, + "y": 1, + "w": 6, + "h": 5 + } + }, + { + "type": "stat", + "title": "Inlined share", + "description": "Commands that rode inside another request's argv.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * sum(increase(tmux_inlined_total{tmux_lane=~\"$lane\"}[$__range])) / clamp_min(sum(increase(tmux_commands_total{tmux_lane=~\"$lane\"}[$__range])), 1)", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "id": 4, + "gridPos": { + "x": 12, + "y": 1, + "w": 6, + "h": 5 + } + }, + { + "type": "stat", + "title": "Failure share", + "description": "Requests tmux rejected.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * sum(increase(tmux_failures_total{tmux_lane=~\"$lane\"}[$__range])) / clamp_min(sum(increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range])), 1)", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "id": 5, + "gridPos": { + "x": 18, + "y": 1, + "w": 6, + "h": 5 + } + }, + { + "type": "row", + "title": "Throughput and latency", + "collapsed": false, + "id": 6, + "gridPos": { + "x": 0, + "y": 6, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Request rate by transport", + "description": "How much each transport is carrying.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_lane) (rate(tmux_requests_total{tmux_lane=~\"$lane\"}[$__rate_interval]))", + "legendFormat": "{{tmux_lane}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 7, + "gridPos": { + "x": 0, + "y": 7, + "w": 12, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "p95 latency by transport", + "description": "Per-request time inside the engine. Click an exemplar to open its trace.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, tmux_lane) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__rate_interval])))", + "legendFormat": "{{tmux_lane}}", + "range": true, + "instant": false, + "refId": "A", + "exemplar": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "s", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 8, + "gridPos": { + "x": 12, + "y": 7, + "w": 12, + "h": 8 + } + }, + { + "type": "heatmap", + "title": "Latency distribution", + "description": "Where requests actually land, not just the tail.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (le) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__rate_interval]))", + "legendFormat": "{{le}}", + "range": true, + "instant": false, + "refId": "A", + "format": "heatmap" + } + ], + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "mode": "scheme", + "scheme": "Spectral", + "steps": 64 + }, + "yAxis": { + "unit": "s" + }, + "legend": { + "show": true + }, + "tooltip": { + "show": true, + "yHistogram": true + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": {} + } + }, + "overrides": [] + }, + "id": 9, + "gridPos": { + "x": 0, + "y": 15, + "w": 12, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "Failures by transport", + "description": "tmux rejections; the smoke workload issues these on purpose.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_lane) (rate(tmux_failures_total{tmux_lane=~\"$lane\"}[$__rate_interval]))", + "legendFormat": "{{tmux_lane}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 10, + "gridPos": { + "x": 12, + "y": 15, + "w": 12, + "h": 8 + } + }, + { + "type": "row", + "title": "Traces, logs, and profiles", + "collapsed": false, + "id": 11, + "gridPos": { + "x": 0, + "y": 23, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "table", + "title": "Requests that batched commands", + "description": "Spans carrying more than one tmux command.", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{ resource.service.name=\"libtmux-engines\" && span.tmux.inlined > 0 }", + "limit": 20, + "tableType": "spans", + "refId": "A" + } + ], + "options": { + "showHeader": true + }, + "id": 12, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + } + }, + { + "type": "logs", + "title": "Engine logs", + "description": "Application logs, correlated to traces by trace_id.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{service_name=\"libtmux-engines\"} | json", + "legendFormat": "__auto", + "range": true, + "instant": false, + "refId": "A" + } + ], + "options": { + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true, + "enableLogDetails": true + }, + "id": 13, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + } + }, + { + "type": "flamegraph", + "title": "CPU profile", + "description": "Where Python time went while the workload ran.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "profile", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": "{service_name=\"libtmux-engines\"}", + "groupBy": [], + "refId": "A" + } + ], + "options": {}, + "id": 14, + "gridPos": { + "x": 0, + "y": 32, + "w": 24, + "h": 11 + } + } + ] +} diff --git a/scripts/lgtm/dashboards/libtmux-transports.json b/scripts/lgtm/dashboards/libtmux-transports.json new file mode 100644 index 000000000..c727364b2 --- /dev/null +++ b/scripts/lgtm/dashboards/libtmux-transports.json @@ -0,0 +1,543 @@ +{ + "uid": "libtmux-transports", + "title": "libtmux / Transports", + "description": "Subprocess against control mode, sync against async. Same operations, different dispatch cost.", + "tags": [ + "libtmux", + "generated" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "lane", + "label": "Transport", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(tmux_requests_total, tmux_lane)", + "refId": "var-lane" + }, + "refresh": 2, + "sort": 1, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "links": [ + { + "title": "libtmux dashboards", + "type": "dashboards", + "tags": [ + "libtmux" + ], + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "icon": "external link" + } + ], + "annotations": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "Share of work", + "collapsed": false, + "id": 1, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "piechart", + "title": "Requests by transport", + "description": "Which transport carried the run.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_lane) (increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range]))", + "legendFormat": "{{tmux_lane}}", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "id": 2, + "gridPos": { + "x": 0, + "y": 1, + "w": 8, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "Commands per request", + "description": "Above 1 means requests are carrying command groups.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_lane) (increase(tmux_commands_total{tmux_lane=~\"$lane\"}[$__range])) / clamp_min(sum by (tmux_lane) (increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range])), 1)", + "legendFormat": "{{tmux_lane}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 3, + "gridPos": { + "x": 8, + "y": 1, + "w": 8, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "Inlined commands", + "description": "Commands that cost no dispatch of their own.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_lane) (rate(tmux_inlined_total{tmux_lane=~\"$lane\"}[$__rate_interval]))", + "legendFormat": "{{tmux_lane}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 4, + "gridPos": { + "x": 16, + "y": 1, + "w": 8, + "h": 8 + } + }, + { + "type": "row", + "title": "Latency percentiles", + "collapsed": false, + "id": 5, + "gridPos": { + "x": 0, + "y": 9, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "p50 by transport", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, tmux_lane) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__rate_interval])))", + "legendFormat": "{{tmux_lane}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "s", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 6, + "gridPos": { + "x": 0, + "y": 10, + "w": 8, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "p95 by transport", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, tmux_lane) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__rate_interval])))", + "legendFormat": "{{tmux_lane}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "s", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 7, + "gridPos": { + "x": 8, + "y": 10, + "w": 8, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "p99 by transport", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, tmux_lane) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__rate_interval])))", + "legendFormat": "{{tmux_lane}}", + "range": true, + "instant": false, + "refId": "A", + "exemplar": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "s", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 8, + "gridPos": { + "x": 16, + "y": 10, + "w": 8, + "h": 8 + } + }, + { + "type": "row", + "title": "Per-transport detail", + "collapsed": false, + "id": 9, + "gridPos": { + "x": 0, + "y": 18, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "table", + "title": "Transport summary", + "description": "Totals for the selected window, per transport.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_lane) (increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range]))", + "legendFormat": "requests", + "range": false, + "instant": true, + "refId": "A", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_lane) (increase(tmux_commands_total{tmux_lane=~\"$lane\"}[$__range]))", + "legendFormat": "commands", + "range": false, + "instant": true, + "refId": "B", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (tmux_lane) (increase(tmux_inlined_total{tmux_lane=~\"$lane\"}[$__range]))", + "legendFormat": "inlined", + "range": false, + "instant": true, + "refId": "C", + "format": "table" + } + ], + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true, + "instance": true + } + } + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto" + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "footer": { + "show": false + } + }, + "id": 10, + "gridPos": { + "x": 0, + "y": 19, + "w": 24, + "h": 8 + } + } + ] +} diff --git a/scripts/lgtm/generate_dashboards.py b/scripts/lgtm/generate_dashboards.py new file mode 100644 index 000000000..12160ab63 --- /dev/null +++ b/scripts/lgtm/generate_dashboards.py @@ -0,0 +1,843 @@ +"""Generate the libtmux Grafana dashboard suite. + +Dashboards are generated rather than hand-edited JSON. A Grafana board is a few +hundred lines of deeply nested objects in which a panel's position is manual +arithmetic, so hand-maintaining six of them guarantees drift. Here a board is a +list of panel calls and :class:`Board` does the grid math. + +The generated JSON is committed. ``scripts/lgtm/up.sh`` regenerates it on every +start and ``tests/test_lgtm_dashboards.py`` fails if the committed copy differs, +so the two cannot silently diverge. + +Every panel must be backed by telemetry ``scripts/otel_smoke.py`` actually +emits. ``scripts/otel_acceptance.py`` executes each panel's own queries and +fails on any that returns nothing, which is what keeps a board honest. +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import typing as t + +PROM: dict[str, str] = {"type": "prometheus", "uid": "prometheus"} +LOKI: dict[str, str] = {"type": "loki", "uid": "loki"} +TEMPO: dict[str, str] = {"type": "tempo", "uid": "tempo"} +PYROSCOPE: dict[str, str] = {"type": "grafana-pyroscope-datasource", "uid": "pyroscope"} + +SERVICE = "libtmux-engines" +# Every panel filters by the lane template variable, so one board serves both +# "all transports together" and "this transport alone". +LANE = 'tmux_lane=~"$lane"' +TAGS = ["libtmux", "generated"] + +BUCKET = "tmux_command_duration_seconds_bucket" + + +def rate_by(label: str, metric: str) -> str: + """Per-second rate of *metric*, grouped by *label*.""" + return f"sum by ({label}) (rate({metric}{{{LANE}}}[$__rate_interval]))" + + +def window_total(metric: str, label: str | None = None) -> str: + """Total increase of *metric* across the dashboard's time range. + + Deliberately windowed rather than a bare counter read. The workload is + short-lived, so a few minutes after it exits Prometheus marks its series + stale and an instant query at ``now`` returns nothing at all -- a stat + panel reading the counter directly goes blank while the timeseries beside + it still shows the run. ``increase`` over ``$__range`` asks what happened + in the window the viewer selected, which is both what they meant and + immune to staleness. + """ + inner = f"increase({metric}{{{LANE}}}[$__range])" + return f"sum by ({label}) ({inner})" if label else f"sum({inner})" + + +def window_quantile(quantile: float, label: str) -> str: + """Latency *quantile* across the whole window, for summary tables.""" + return ( + f"histogram_quantile({quantile}, sum by (le, {label}) " + f"(rate({BUCKET}{{{LANE}}}[$__range])))" + ) + + +def quantile_by(quantile: float, label: str) -> str: + """Latency *quantile* from the duration histogram, grouped by *label*.""" + return ( + f"histogram_quantile({quantile}, sum by (le, {label}) " + f"(rate({BUCKET}{{{LANE}}}[$__rate_interval])))" + ) + + +ERR_THRESHOLDS = [ + {"color": "green", "value": None}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 5}, +] + + +def target( + expr: str, + legend: str = "", + *, + exemplar: bool = False, + fmt: str = "time_series", + instant: bool = False, + ref: str = "A", + datasource: dict[str, str] | None = None, +) -> dict[str, t.Any]: + """Build one query target. + + Parameters + ---------- + expr : str + The query, in the datasource's own language. + legend : str + Legend format; Grafana expands ``{{label}}``. + exemplar : bool + Overlay exemplars, giving the metric-to-trace pivot. + fmt : str + ``time_series``, ``heatmap``, or ``table``. + instant : bool + Ask for a single point instead of a range. + ref : str + Query id, unique within a panel. + datasource : dict or None + Defaults to Prometheus. + + Returns + ------- + dict + A Grafana target object. + """ + tgt: dict[str, t.Any] = { + "datasource": datasource or PROM, + "editorMode": "code", + "expr": expr, + "legendFormat": legend or "__auto", + "range": not instant, + "instant": instant, + "refId": ref, + } + if exemplar: + tgt["exemplar"] = True + if fmt != "time_series": + tgt["format"] = fmt + return tgt + + +class Board: + """A dashboard that lays itself out on Grafana's 24-column grid.""" + + def __init__( + self, + uid: str, + title: str, + *, + description: str = "", + refresh: str = "30s", + time_from: str = "now-1h", + ) -> None: + self.uid = uid + self.title = title + self.description = description + self.refresh = refresh + self.time_from = time_from + self._panels: list[dict[str, t.Any]] = [] + self._templates: list[dict[str, t.Any]] = [] + self._id = 0 + self._x = 0 + self._y = 0 + self._row_h = 0 + + def _next_id(self) -> int: + self._id += 1 + return self._id + + def _place(self, w: int, h: int) -> dict[str, int]: + if self._x + w > 24: + self._x = 0 + self._y += self._row_h + self._row_h = 0 + pos = {"x": self._x, "y": self._y, "w": w, "h": h} + self._x += w + self._row_h = max(self._row_h, h) + return pos + + def row(self, title: str) -> None: + """Start a labelled row, flushing the current one.""" + if self._x != 0: + self._x = 0 + self._y += self._row_h + self._row_h = 0 + self._panels.append( + { + "type": "row", + "title": title, + "collapsed": False, + "id": self._next_id(), + "gridPos": {"x": 0, "y": self._y, "w": 24, "h": 1}, + "panels": [], + } + ) + self._y += 1 + + def add(self, panel: dict[str, t.Any], *, w: int = 12, h: int = 8) -> None: + """Place a panel at the current grid cursor.""" + panel["id"] = self._next_id() + panel["gridPos"] = self._place(w, h) + self._panels.append(panel) + + def lane_variable(self) -> None: + """Add the transport selector every board filters on.""" + self._templates.append( + { + "name": "lane", + "label": "Transport", + "type": "query", + "datasource": PROM, + "query": { + "qryType": 1, + "query": "label_values(tmux_requests_total, tmux_lane)", + "refId": "var-lane", + }, + "refresh": 2, + "sort": 1, + "includeAll": True, + "allValue": ".*", + "multi": True, + "current": {"text": "All", "value": "$__all"}, + } + ) + + def to_dict(self) -> dict[str, t.Any]: + """Render the dashboard envelope.""" + return { + "uid": self.uid, + "title": self.title, + "description": self.description, + "tags": TAGS, + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": True, + "graphTooltip": 1, + "refresh": self.refresh, + "time": {"from": self.time_from, "to": "now"}, + "templating": {"list": self._templates}, + "links": [ + { + "title": "libtmux dashboards", + "type": "dashboards", + "tags": ["libtmux"], + "asDropdown": True, + "includeVars": True, + "keepTime": True, + "icon": "external link", + } + ], + "annotations": {"list": []}, + "panels": self._panels, + } + + +# --------------------------------------------------------------------------- +# Panel builders. +# --------------------------------------------------------------------------- +def timeseries( + title: str, + targets: list[dict[str, t.Any]], + *, + unit: str = "short", + description: str = "", + stacking: bool = False, +) -> dict[str, t.Any]: + """Build a timeseries panel.""" + custom: dict[str, t.Any] = { + "fillOpacity": 18 if stacking else 10, + "showPoints": "never", + "lineWidth": 2, + } + if stacking: + custom["stacking"] = {"mode": "normal", "group": "A"} + return { + "type": "timeseries", + "title": title, + "description": description, + "datasource": PROM, + "targets": targets, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "unit": unit, + "custom": custom, + }, + "overrides": [], + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["lastNotNull", "max"], + }, + "tooltip": {"mode": "multi", "sort": "desc"}, + }, + } + + +def stat( + title: str, + targets: list[dict[str, t.Any]], + *, + unit: str = "short", + description: str = "", + thresholds: list[dict[str, t.Any]] | None = None, +) -> dict[str, t.Any]: + """Build a single-value stat panel.""" + field: dict[str, t.Any] = {"unit": unit} + if thresholds is not None: + field["color"] = {"mode": "thresholds"} + field["thresholds"] = {"mode": "absolute", "steps": thresholds} + else: + field["color"] = {"mode": "palette-classic"} + return { + "type": "stat", + "title": title, + "description": description, + "datasource": PROM, + "targets": targets, + "fieldConfig": {"defaults": field, "overrides": []}, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "textMode": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + }, + } + + +def heatmap(title: str, expr: str, *, description: str = "") -> dict[str, t.Any]: + """Build a latency heatmap from histogram buckets.""" + return { + "type": "heatmap", + "title": title, + "description": description, + "datasource": PROM, + "targets": [target(expr, "{{le}}", fmt="heatmap")], + "options": { + "calculate": False, + "cellGap": 1, + "color": {"mode": "scheme", "scheme": "Spectral", "steps": 64}, + "yAxis": {"unit": "s"}, + "legend": {"show": True}, + "tooltip": {"show": True, "yHistogram": True}, + }, + "fieldConfig": {"defaults": {"custom": {"hideFrom": {}}}, "overrides": []}, + } + + +def piechart( + title: str, targets: list[dict[str, t.Any]], *, description: str = "" +) -> dict[str, t.Any]: + """Build a pie chart of proportions.""" + return { + "type": "piechart", + "title": title, + "description": description, + "datasource": PROM, + "targets": targets, + "fieldConfig": { + "defaults": {"color": {"mode": "palette-classic"}, "unit": "short"}, + "overrides": [], + }, + "options": { + "displayLabels": ["percent"], + "legend": { + "displayMode": "table", + "placement": "right", + "values": ["value"], + }, + "pieType": "donut", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + }, + } + + +def table( + title: str, targets: list[dict[str, t.Any]], *, description: str = "" +) -> dict[str, t.Any]: + """Build a table panel from instant queries.""" + return { + "type": "table", + "title": title, + "description": description, + "datasource": PROM, + "targets": targets, + "transformations": [ + {"id": "merge", "options": {}}, + { + "id": "organize", + "options": { + "excludeByName": {"Time": True, "job": True, "instance": True} + }, + }, + ], + "fieldConfig": { + "defaults": {"color": {"mode": "thresholds"}, "custom": {"align": "auto"}}, + "overrides": [], + }, + "options": {"showHeader": True, "footer": {"show": False}}, + } + + +def logs(title: str, expr: str, *, description: str = "") -> dict[str, t.Any]: + """Build a Loki logs panel.""" + return { + "type": "logs", + "title": title, + "description": description, + "datasource": LOKI, + "targets": [target(expr, datasource=LOKI, ref="A")], + "options": { + "showTime": True, + "sortOrder": "Descending", + "wrapLogMessage": True, + "enableLogDetails": True, + }, + } + + +def flamegraph(title: str, *, description: str = "") -> dict[str, t.Any]: + """Build a Pyroscope flame graph of the profiled process.""" + return { + "type": "flamegraph", + "title": title, + "description": description, + "datasource": PYROSCOPE, + "targets": [ + { + "datasource": PYROSCOPE, + "queryType": "profile", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": f'{{service_name="{SERVICE}"}}', + "groupBy": [], + "refId": "A", + } + ], + "options": {}, + } + + +def traces(title: str, query: str, *, description: str = "") -> dict[str, t.Any]: + """Build a Tempo search panel.""" + return { + "type": "table", + "title": title, + "description": description, + "datasource": TEMPO, + "targets": [ + { + "datasource": TEMPO, + "queryType": "traceql", + "query": query, + "limit": 20, + "tableType": "spans", + "refId": "A", + } + ], + "options": {"showHeader": True}, + } + + +# --------------------------------------------------------------------------- +# Boards. +# --------------------------------------------------------------------------- +def build_overview() -> Board: + """Build the board answering whether tmux work is flowing and healthy.""" + board = Board( + "libtmux-overview", + "libtmux / Overview", + description=( + "Throughput, latency, and failures across every engine transport. " + "Metric panels carry exemplars, so a latency spike links to the " + "trace behind it." + ), + ) + board.lane_variable() + + board.row("Totals") + board.add( + stat( + "Requests", + [target(window_total("tmux_requests_total"), instant=True)], + description="Requests dispatched to an engine.", + ), + w=6, + h=5, + ) + board.add( + stat( + "tmux commands", + [target(window_total("tmux_commands_total"), instant=True)], + description="Commands tmux was told to run; a group counts as its members.", + ), + w=6, + h=5, + ) + board.add( + stat( + "Inlined share", + [ + target( + f"100 * {window_total('tmux_inlined_total')} " + f"/ clamp_min({window_total('tmux_commands_total')}, 1)", + instant=True, + ) + ], + unit="percent", + description="Commands that rode inside another request's argv.", + ), + w=6, + h=5, + ) + board.add( + stat( + "Failure share", + [ + target( + f"100 * {window_total('tmux_failures_total')} " + f"/ clamp_min({window_total('tmux_requests_total')}, 1)", + instant=True, + ) + ], + unit="percent", + thresholds=ERR_THRESHOLDS, + description="Requests tmux rejected.", + ), + w=6, + h=5, + ) + + board.row("Throughput and latency") + board.add( + timeseries( + "Request rate by transport", + [ + target( + rate_by("tmux_lane", "tmux_requests_total"), + "{{tmux_lane}}", + ) + ], + unit="reqps", + description="How much each transport is carrying.", + ) + ) + board.add( + timeseries( + "p95 latency by transport", + [ + target( + quantile_by(0.95, "tmux_lane"), + "{{tmux_lane}}", + exemplar=True, + ) + ], + unit="s", + description=( + "Per-request time inside the engine. " + "Click an exemplar to open its trace." + ), + ) + ) + board.add( + heatmap( + "Latency distribution", + rate_by("le", BUCKET), + description="Where requests actually land, not just the tail.", + ) + ) + board.add( + timeseries( + "Failures by transport", + [ + target( + rate_by("tmux_lane", "tmux_failures_total"), + "{{tmux_lane}}", + ) + ], + unit="reqps", + description="tmux rejections; the smoke workload issues these on purpose.", + ) + ) + + board.row("Traces, logs, and profiles") + board.add( + traces( + "Requests that batched commands", + '{ resource.service.name="libtmux-engines" && span.tmux.inlined > 0 }', + description="Spans carrying more than one tmux command.", + ) + ) + board.add( + logs( + "Engine logs", + '{service_name="libtmux-engines"} | json', + description="Application logs, correlated to traces by trace_id.", + ) + ) + board.add( + flamegraph( + "CPU profile", + description="Where Python time went while the workload ran.", + ), + w=24, + h=11, + ) + return board + + +def build_transports() -> Board: + """Build the board comparing transports against each other.""" + board = Board( + "libtmux-transports", + "libtmux / Transports", + description=( + "Subprocess against control mode, sync against async. Same " + "operations, different dispatch cost." + ), + ) + board.lane_variable() + + board.row("Share of work") + board.add( + piechart( + "Requests by transport", + [ + target( + window_total("tmux_requests_total", "tmux_lane"), + "{{tmux_lane}}", + instant=True, + ) + ], + description="Which transport carried the run.", + ), + w=8, + h=8, + ) + board.add( + timeseries( + "Commands per request", + [ + target( + f"{window_total('tmux_commands_total', 'tmux_lane')} " + f"/ clamp_min(" + f"{window_total('tmux_requests_total', 'tmux_lane')}, 1)", + "{{tmux_lane}}", + ) + ], + description="Above 1 means requests are carrying command groups.", + ), + w=8, + h=8, + ) + board.add( + timeseries( + "Inlined commands", + [ + target( + rate_by("tmux_lane", "tmux_inlined_total"), + "{{tmux_lane}}", + ) + ], + unit="reqps", + description="Commands that cost no dispatch of their own.", + ), + w=8, + h=8, + ) + + board.row("Latency percentiles") + for quantile, label in ((0.5, "p50"), (0.95, "p95"), (0.99, "p99")): + board.add( + timeseries( + f"{label} by transport", + [ + target( + quantile_by(quantile, "tmux_lane"), + "{{tmux_lane}}", + exemplar=quantile == 0.99, + ) + ], + unit="s", + ), + w=8, + h=8, + ) + + board.row("Per-transport detail") + board.add( + table( + "Transport summary", + [ + target( + window_total("tmux_requests_total", "tmux_lane"), + "requests", + fmt="table", + instant=True, + ref="A", + ), + target( + window_total("tmux_commands_total", "tmux_lane"), + "commands", + fmt="table", + instant=True, + ref="B", + ), + target( + window_total("tmux_inlined_total", "tmux_lane"), + "inlined", + fmt="table", + instant=True, + ref="C", + ), + ], + description="Totals for the selected window, per transport.", + ), + w=24, + h=8, + ) + return board + + +def build_commands() -> Board: + """Build the board breaking work down by tmux command.""" + board = Board( + "libtmux-commands", + "libtmux / Commands", + description="Which tmux commands the workload issues, and what they cost.", + ) + board.lane_variable() + + board.row("Command mix") + board.add( + piechart( + "Requests by command", + [ + target( + window_total("tmux_requests_total", "tmux_command"), + "{{tmux_command}}", + instant=True, + ) + ], + ), + w=8, + h=9, + ) + board.add( + timeseries( + "Request rate by command", + [ + target( + rate_by("tmux_command", "tmux_requests_total"), + "{{tmux_command}}", + ) + ], + unit="reqps", + stacking=True, + ), + w=16, + h=9, + ) + + board.row("Cost and failures") + board.add( + timeseries( + "p95 by command", + [ + target( + quantile_by(0.95, "tmux_command"), + "{{tmux_command}}", + ) + ], + unit="s", + ) + ) + board.add( + timeseries( + "Failures by command", + [ + target( + rate_by("tmux_command", "tmux_failures_total"), + "{{tmux_command}}", + ) + ], + unit="reqps", + ) + ) + board.add( + table( + "Command summary", + [ + target( + window_total("tmux_requests_total", "tmux_command"), + "requests", + fmt="table", + instant=True, + ref="A", + ), + target( + window_total("tmux_commands_total", "tmux_command"), + "commands", + fmt="table", + instant=True, + ref="B", + ), + target( + window_quantile(0.95, "tmux_command"), + "p95", + fmt="table", + instant=True, + ref="C", + ), + ], + ), + w=24, + h=8, + ) + return board + + +BUILDERS = (build_overview, build_transports, build_commands) + + +def write_dashboards(out_dir: pathlib.Path) -> list[pathlib.Path]: + """Write every board to *out_dir*, returning the paths written.""" + out_dir.mkdir(parents=True, exist_ok=True) + written = [] + for builder in BUILDERS: + board = builder() + path = out_dir / f"{board.uid}.json" + path.write_text(json.dumps(board.to_dict(), indent=2) + "\n", encoding="utf-8") + written.append(path) + return written + + +def main(argv: list[str] | None = None) -> int: + """Regenerate the dashboard suite.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output", + type=pathlib.Path, + default=pathlib.Path(__file__).parent / "dashboards", + ) + args = parser.parse_args(argv) + for path in write_dashboards(args.output): + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/lgtm/grafana-dashboards-libtmux.yaml b/scripts/lgtm/grafana-dashboards-libtmux.yaml new file mode 100644 index 000000000..9d1b5a94d --- /dev/null +++ b/scripts/lgtm/grafana-dashboards-libtmux.yaml @@ -0,0 +1,16 @@ +apiVersion: 1 + +# Registers the libtmux dashboard suite next to otel-lgtm's built-in providers +# (Grafana loads every *.yaml in this directory). The JSON is generated by +# scripts/lgtm/generate_dashboards.py, which up.sh runs on startup, and mounted +# read-only at the path below. +providers: + - name: libtmux + type: file + folder: libtmux + disableDeletion: false + allowUiUpdates: true + updateIntervalSeconds: 30 + options: + path: /otel-lgtm/dashboards-libtmux + foldersFromFilesStructure: false diff --git a/scripts/lgtm/grafana-datasources.yaml b/scripts/lgtm/grafana-datasources.yaml new file mode 100644 index 000000000..f86230a01 --- /dev/null +++ b/scripts/lgtm/grafana-datasources.yaml @@ -0,0 +1,66 @@ +# Repo-owned copy of the otel-lgtm image's datasource provisioning, mounted by +# scripts/lgtm/up.sh. +# +# It is committed unchanged, and the reason is the uids: every panel in +# scripts/lgtm/dashboards/ binds to "prometheus", "tempo", "loki", or +# "pyroscope" by uid. Pinning the file alongside the pinned image means a board +# cannot silently lose its datasource because an image bumped a uid. +# +# The exemplar and traces-to-logs wiring below is what lets a metric panel jump +# to the trace behind a spike, and a trace jump to its logs. +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + url: http://127.0.0.1:9090 + editable: true + jsonData: + timeInterval: 60s + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + urlDisplayLabel: "Trace: $${__value.raw}" + + - name: Tempo + type: tempo + uid: tempo + url: http://127.0.0.1:3200 + editable: true + jsonData: + tracesToLogsV2: + customQuery: true + datasourceUid: "loki" + query: '{$${__tags}} | trace_id = "$${__trace.traceId}"' + tags: + - key: "service.name" + value: "service_name" + + serviceMap: + datasourceUid: "prometheus" + search: + hide: false + nodeGraph: + enabled: true + lokiSearch: + datasourceUid: "loki" + + - name: Loki + type: loki + uid: loki + url: http://127.0.0.1:3100 + editable: true + jsonData: + derivedFields: + - name: "trace_id" + matcherType: "label" + matcherRegex: "trace_id" + url: "$${__value.raw}" + datasourceUid: "tempo" + urlDisplayLabel: "Trace: $${__value.raw}" + + - name: Pyroscope + type: grafana-pyroscope-datasource + uid: pyroscope + url: http://127.0.0.1:4040 diff --git a/scripts/lgtm/telemetry.py b/scripts/lgtm/telemetry.py new file mode 100644 index 000000000..3551d353d --- /dev/null +++ b/scripts/lgtm/telemetry.py @@ -0,0 +1,238 @@ +"""OpenTelemetry exporters for the engine instrumentation seam. + +These are sinks, in the sense +:mod:`libtmux.experimental.engines.instrumentation` means it: they observe +commands through :func:`~libtmux.experimental.engines.instrumentation.instrument` +and the engines themselves are never touched. Nothing here is imported by +libtmux, so the library keeps no OpenTelemetry dependency. + +One sink emits both spans and metrics rather than two sinks emitting one each, +because the histogram must record while its span is current. That is what +attaches an exemplar, and the exemplar is what lets a Grafana panel jump from a +latency spike to the trace that caused it. +""" + +from __future__ import annotations + +import time +import typing as t + +from opentelemetry import metrics, trace +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +from libtmux.experimental.engines.control_mode import command_count + +if t.TYPE_CHECKING: + from libtmux.experimental.engines.base import CommandRequest, CommandResult + +SERVICE_NAME = "libtmux-engines" + +# Latency buckets in seconds. A control-mode command is tens of microseconds and +# a subprocess command is a few milliseconds, so the buckets have to span four +# orders of magnitude or one transport lands entirely in the first bucket. +DURATION_BUCKETS = ( + 0.0001, + 0.00025, + 0.0005, + 0.001, + 0.0025, + 0.005, + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, +) + + +class OTelSink: + """Emit one span and one set of metric points per tmux command. + + Attributes + ---------- + tracer : opentelemetry.trace.Tracer + Tracer used for per-command spans. + lane : str + Transport label attached to every metric point. + """ + + __slots__ = ( + "_commands", + "_duration", + "_failures", + "_inlined", + "_lane", + "_requests", + "_tracer", + ) + + def __init__(self, tracer: t.Any, meter: t.Any, lane: str) -> None: + self._tracer = tracer + self._lane = lane + self._requests = meter.create_counter( + "tmux.requests", description="Requests dispatched to an engine." + ) + self._commands = meter.create_counter( + "tmux.commands", description="tmux commands those requests carried." + ) + self._inlined = meter.create_counter( + "tmux.inlined", + description="Commands that rode inside another request's argv.", + ) + self._failures = meter.create_counter( + "tmux.failures", description="Commands tmux rejected." + ) + self._duration = meter.create_histogram( + "tmux.command.duration", + unit="s", + description="Wall time spent inside the engine per request.", + explicit_bucket_boundaries_advisory=DURATION_BUCKETS, + ) + + def _attrs(self, command: str) -> dict[str, str]: + return {"tmux.lane": self._lane, "tmux.command": command} + + def before_command(self, request: CommandRequest) -> tuple[t.Any, float, str]: + """Open a span and count the request, returning per-command state.""" + argv = tuple(str(arg) for arg in request.args) + command = argv[0] if argv else "unknown" + commands = command_count(tuple(request.args)) + attrs = self._attrs(command) + + span = self._tracer.start_span(f"tmux {command}") + span.set_attribute("tmux.command", command) + span.set_attribute("tmux.lane", self._lane) + span.set_attribute("tmux.statement", " ".join(argv)[:512]) + span.set_attribute("tmux.commands", commands) + span.set_attribute("tmux.inlined", commands - 1) + + self._requests.add(1, attrs) + self._commands.add(commands, attrs) + self._inlined.add(commands - 1, attrs) + return span, time.perf_counter(), command + + def after_command( + self, request: CommandRequest, result: CommandResult, state: t.Any + ) -> None: + """Close the span and record its duration with an exemplar.""" + del request + span, started, command = state + attrs = self._attrs(command) + span.set_attribute("tmux.returncode", result.returncode) + if result.returncode != 0: + self._failures.add(1, attrs) + # Recording while the span is current is what attaches the exemplar. + with trace.use_span(span, end_on_exit=False): + self._duration.record(time.perf_counter() - started, attrs) + span.end() + + def handle_error( + self, request: CommandRequest, error: BaseException, state: t.Any + ) -> None: + """Record the failure, then let the exception continue.""" + del request + span, started, command = state + attrs = self._attrs(command) + self._failures.add(1, attrs) + span.record_exception(error) + with trace.use_span(span, end_on_exit=False): + self._duration.record(time.perf_counter() - started, attrs) + span.end() + + +class Telemetry(t.NamedTuple): + """Providers and handles for one instrumented process. + + Attributes + ---------- + tracer_provider : opentelemetry.sdk.trace.TracerProvider + Provider owning span export; needs an explicit shutdown. + meter_provider : opentelemetry.sdk.metrics.MeterProvider + Provider owning metric export; needs an explicit shutdown. + logger_provider : opentelemetry.sdk._logs.LoggerProvider + Provider owning log export; needs an explicit shutdown. + tracer : opentelemetry.trace.Tracer + Tracer for per-command spans. + meter : opentelemetry.metrics.Meter + Meter the sinks build instruments from. + handler : logging.Handler + Handler that ships records to Loki with trace context attached. + """ + + tracer_provider: t.Any + meter_provider: t.Any + logger_provider: t.Any + tracer: t.Any + meter: t.Any + handler: t.Any + + def shutdown(self) -> None: + """Flush and stop every provider, in export order.""" + self.tracer_provider.force_flush() + self.meter_provider.force_flush() + self.logger_provider.force_flush() + self.tracer_provider.shutdown() + self.meter_provider.shutdown() + self.logger_provider.shutdown() + + +def build(endpoint: str, *, run_id: str, export_interval_ms: int = 2000) -> Telemetry: + """Wire OTLP exporters for traces, metrics, and logs. + + Parameters + ---------- + endpoint : str + OTLP HTTP base URL, for example ``http://127.0.0.1:4318``. + run_id : str + Identifies one smoke run, so a dashboard can isolate it. + export_interval_ms : int + How often metrics are pushed. Short, because a smoke run is short. + + Returns + ------- + Telemetry + Providers and handles; call :meth:`Telemetry.shutdown` when done. + """ + resource = Resource.create({"service.name": SERVICE_NAME, "libtmux.run_id": run_id}) + + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")) + ) + trace.set_tracer_provider(tracer_provider) + + meter_provider = MeterProvider( + resource=resource, + metric_readers=[ + PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"), + export_interval_millis=export_interval_ms, + ) + ], + ) + metrics.set_meter_provider(meter_provider) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs")) + ) + + return Telemetry( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + tracer=trace.get_tracer("libtmux.engines"), + meter=metrics.get_meter("libtmux.engines"), + handler=LoggingHandler(logger_provider=logger_provider), + ) diff --git a/scripts/lgtm/up.sh b/scripts/lgtm/up.sh new file mode 100755 index 000000000..e37c577ea --- /dev/null +++ b/scripts/lgtm/up.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Start the local Grafana LGTM stack that the telemetry checks query against. +# +# One container runs Grafana, Loki, Tempo, Prometheus, Pyroscope, and an +# OpenTelemetry collector in front of them. The libtmux dashboards are +# regenerated and bind-mounted on every start, so a fresh checkout gets the +# same boards without clicking through the UI. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +CONTAINER="${LIBTMUX_LGTM_CONTAINER:-libtmux-lgtm}" + +# Pin the image rather than tracking :latest, so a rerun months from now sees +# the Prometheus and Pyroscope this stack was verified against. 0.30.2 runs +# Prometheus 3.13 with --web.enable-otlp-receiver and +# --enable-feature=exemplar-storage, which is what makes the metric-to-trace +# pivot in the dashboards work. +IMAGE="${LIBTMUX_LGTM_IMAGE:-docker.io/grafana/otel-lgtm:0.30.2}" + +# Grafana's own default is 3000 and Prometheus's is 9090. Both are commonly +# taken on a dev box, and a taken port does not fail loudly: Docker still +# publishes, but a host process already bound there answers first, so queries +# reach the WRONG server and return plausible data. verify.sh checks for +# exactly that. Override either if these also collide. +GRAFANA_PORT="${LIBTMUX_LGTM_GRAFANA_PORT:-3900}" +PROM_PORT="${LIBTMUX_LGTM_PROM_PORT:-9099}" + +# Bump when the mounted config or the run shape changes, so an existing +# container is recreated rather than restarted with stale mounts. +CONFIG_LABEL="dashboards-v1" + +if [[ -n "${PYTHON:-}" ]]; then + read -r -a python_cmd <<< "$PYTHON" +elif command -v uv > /dev/null 2>&1 && [[ -f "$ROOT/pyproject.toml" ]]; then + python_cmd=(uv run python) +else + python_cmd=(python3) +fi + +"${python_cmd[@]}" "$ROOT/scripts/lgtm/generate_dashboards.py" \ + --output "$ROOT/scripts/lgtm/dashboards" + +docker_run=( + run + -d + --name "$CONTAINER" + --init + --restart unless-stopped + --label "libtmux.lgtm.config=$CONFIG_LABEL" + -p "${GRAFANA_PORT}:3000" + -p 3100:3100 + -p 3200:3200 + -p 4040:4040 + -p 4317:4317 + -p 4318:4318 + -p "${PROM_PORT}:9090" + -v "$ROOT/scripts/lgtm/grafana-datasources.yaml:/otel-lgtm/grafana/conf/provisioning/datasources/grafana-datasources.yaml:ro" + -v "$ROOT/scripts/lgtm/grafana-dashboards-libtmux.yaml:/otel-lgtm/grafana/conf/provisioning/dashboards/libtmux.yaml:ro" + -v "$ROOT/scripts/lgtm/dashboards:/otel-lgtm/dashboards-libtmux:ro" + -e GF_PATHS_DATA=/data/grafana + "$IMAGE" +) + +if docker inspect "$CONTAINER" > /dev/null 2>&1; then + current="$( + docker inspect --format '{{ index .Config.Labels "libtmux.lgtm.config" }}' \ + "$CONTAINER" 2> /dev/null || true + )" + if [[ "$current" != "$CONFIG_LABEL" ]]; then + docker rm -f "$CONTAINER" > /dev/null + docker "${docker_run[@]}" > /dev/null + else + docker start "$CONTAINER" > /dev/null + fi +else + docker "${docker_run[@]}" > /dev/null +fi + +printf 'waiting for the stack' +for _ in $(seq 1 45); do + state="$(docker inspect --format '{{.State.Health.Status}}' "$CONTAINER" 2> /dev/null || echo none)" + if [[ "$state" == "healthy" ]]; then + printf '\n' + break + fi + printf '.' + sleep 4 +done + +cat < str: + """Expand the template variables Grafana would substitute.""" + return ( + expr.replace("$__rate_interval", RATE_INTERVAL) + .replace("$__interval", RATE_INTERVAL) + .replace("$__range", DASHBOARD_RANGE) + .replace("$lane", ".*") + ) + + +def fetch(url: str, params: dict[str, str], *, timeout: float = 30.0) -> dict: + """GET *url* with *params* and decode the JSON body.""" + query = urllib.parse.urlencode(params) + with urllib.request.urlopen(f"{url}?{query}", timeout=timeout) as response: + return json.loads(response.read().decode()) + + +def _finite_series(rows: list[dict], *, ranged: bool) -> int: + """Count series carrying at least one real number. + + ``histogram_quantile`` over empty buckets yields NaN rather than an empty + result, so a series of nothing but NaN carries no data. + """ + found = 0 + for row in rows: + samples = row.get("values", []) if ranged else [row.get("value", [])] + for sample in samples: + if len(sample) < 2: + continue + try: + if math.isfinite(float(sample[1])): + found += 1 + break + except (TypeError, ValueError): + continue + return found + + +def check_prometheus(base: str, expr: str, *, ranged: bool) -> tuple[bool, str]: + """Query Prometheus the way the panel does, and report what came back. + + Honoring the panel's own mode matters. Grafana evaluates a range panel + across the dashboard window, so a ``rate`` panel still draws a line from + samples earlier in the window even when nothing arrived in the last few + minutes. Checking that same panel with an instant query at ``now`` reports + an emptiness the viewer never sees, and checking an instant panel with a + range query would hide one they do. + """ + if ranged: + now = int(time.time()) + params = { + "query": expr, + "start": str(now - LOOKBACK_SECONDS), + "end": str(now), + "step": "60", + } + url = f"{base}/api/v1/query_range" + else: + params = {"query": expr} + url = f"{base}/api/v1/query" + try: + body = fetch(url, params) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + return False, f"{type(error).__name__}: {error}" + if body.get("status") != "success": + return False, str(body.get("error", "query failed"))[:120] + rows = body["data"]["result"] + if not rows: + return False, "no series" + finite = _finite_series(rows, ranged=ranged) + if not finite: + return False, f"{len(rows)} series, all NaN" + return True, f"{finite} series" + + +def check_loki(base: str, expr: str) -> tuple[bool, str]: + """Run a range query and report whether any stream carried entries.""" + now = int(time.time()) + try: + body = fetch( + f"{base}/loki/api/v1/query_range", + { + "query": expr, + "start": f"{now - LOOKBACK_SECONDS}000000000", + "end": f"{now}000000000", + "limit": "50", + }, + ) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + return False, f"{type(error).__name__}: {error}" + streams = body.get("data", {}).get("result", []) + entries = sum(len(s.get("values", [])) for s in streams) + if not entries: + return False, "no log entries" + return True, f"{entries} entries" + + +def check_tempo(base: str, query: str) -> tuple[bool, str]: + """Run a TraceQL search and report whether it matched traces.""" + now = int(time.time()) + try: + body = fetch( + f"{base}/api/search", + { + "q": query, + "limit": "20", + "start": str(now - LOOKBACK_SECONDS), + "end": str(now), + }, + ) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + return False, f"{type(error).__name__}: {error}" + found = body.get("traces") or [] + if not found: + return False, "no traces" + return True, f"{len(found)} traces" + + +def check_pyroscope(base: str, profile_type: str, selector: str) -> tuple[bool, str]: + """Render a flame graph and report whether it carried frames.""" + now = int(time.time()) + try: + body = fetch( + f"{base}/pyroscope/render", + { + "query": f"{profile_type}{selector}", + "from": str(now - LOOKBACK_SECONDS), + "until": str(now), + "format": "json", + }, + ) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + return False, f"{type(error).__name__}: {error}" + names = body.get("flamebearer", {}).get("names", []) + # A response with only the synthetic "total" root carries no samples. + if len(names) <= 1: + return False, "no profile samples" + return True, f"{len(names)} frames" + + +def check_panel(panel: dict, board: str, endpoints: Endpoints) -> list[Result]: + """Verify every target on one panel.""" + results: list[Result] = [] + title = panel.get("title", "") + for tgt in panel.get("targets", []): + kind = (tgt.get("datasource") or {}).get("type", "prometheus") + if kind == "prometheus": + # Grafana runs a target as a range query unless it is marked + # instant; the check has to make the same choice. + ranged = bool(tgt.get("range", not tgt.get("instant", False))) + ok, detail = check_prometheus( + endpoints.prometheus, expand(tgt["expr"]), ranged=ranged + ) + elif kind == "loki": + ok, detail = check_loki(endpoints.loki, expand(tgt["expr"])) + elif kind == "tempo": + ok, detail = check_tempo(endpoints.tempo, expand(tgt["query"])) + elif kind == "grafana-pyroscope-datasource": + ok, detail = check_pyroscope( + endpoints.pyroscope, tgt["profileTypeId"], expand(tgt["labelSelector"]) + ) + else: + ok, detail = False, f"unknown datasource {kind}" + results.append(Result(board, title, kind, ok, detail)) + return results + + +def check_dashboards(endpoints: Endpoints) -> list[Result]: + """Verify every panel of every generated dashboard, once.""" + results: list[Result] = [] + for path in sorted(DASHBOARDS.glob("*.json")): + board = json.loads(path.read_text(encoding="utf-8")) + for panel in board["panels"]: + if panel["type"] == "row": + continue + results.extend(check_panel(panel, board["uid"], endpoints)) + return results + + +def check_until(endpoints: Endpoints, timeout: float, poll: float) -> list[Result]: + """Re-check until everything has data or *timeout* expires. + + Ingestion is asynchronous and each backend buffers on its own schedule, so + "no data yet" and "no data ever" look identical at any single instant. A + cold-started Tempo in particular can take longer to make a just-written + trace searchable than Prometheus takes to expose a metric. + + Polling removes that race without inflating a fixed sleep for everyone: + a warm stack returns on the first pass, and a cold one waits only as long + as it actually needs. + """ + deadline = time.monotonic() + timeout + results = check_dashboards(endpoints) + while any(not result.ok for result in results) and time.monotonic() < deadline: + pending = sum(1 for result in results if not result.ok) + print(f" waiting for {pending} panel(s) to receive data...") + time.sleep(poll) + results = check_dashboards(endpoints) + return results + + +def main(argv: list[str] | None = None) -> int: + """Run the acceptance sweep and print a per-panel report.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prometheus", default="http://127.0.0.1:9099") + parser.add_argument("--loki", default="http://127.0.0.1:3100") + parser.add_argument("--tempo", default="http://127.0.0.1:3200") + parser.add_argument("--pyroscope", default="http://127.0.0.1:4040") + parser.add_argument( + "--start-stack", action="store_true", help="run scripts/lgtm/up.sh first" + ) + parser.add_argument( + "--smoke", action="store_true", help="run scripts/otel_smoke.py first" + ) + parser.add_argument( + "--settle", + type=float, + default=5.0, + help="seconds to wait before the first query", + ) + parser.add_argument( + "--timeout", + type=float, + default=120.0, + help="seconds to keep re-checking panels that have no data yet", + ) + parser.add_argument( + "--poll", + type=float, + default=10.0, + help="seconds between re-checks", + ) + args = parser.parse_args(argv) + + if args.start_stack: + subprocess.run([str(ROOT / "scripts" / "lgtm" / "up.sh")], check=True) + if args.smoke: + subprocess.run( + [sys.executable, str(ROOT / "scripts" / "otel_smoke.py")], check=True + ) + if args.start_stack or args.smoke: + time.sleep(args.settle) + + endpoints = Endpoints(args.prometheus, args.loki, args.tempo, args.pyroscope) + results = check_until(endpoints, args.timeout, args.poll) + + width = max((len(r.panel) for r in results), default=10) + current = "" + for result in results: + if result.board != current: + current = result.board + print(f"\n{current}") + mark = "ok " if result.ok else "EMPTY" + print(f" {mark} {result.panel:<{width}} {result.kind:<28} {result.detail}") + + failed = [r for r in results if not r.ok] + print(f"\n{len(results) - len(failed)}/{len(results)} panel queries returned data") + if failed: + print("panels with no data:") + for result in failed: + print(f" {result.board}/{result.panel}: {result.detail}") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/otel_smoke.py b/scripts/otel_smoke.py new file mode 100644 index 000000000..aaa1d2a43 --- /dev/null +++ b/scripts/otel_smoke.py @@ -0,0 +1,275 @@ +"""Drive a real tmux workload through the engine seam into the local LGTM stack. + +Emits all four signals from one run: spans and metrics per tmux command, logs +carrying trace context, and a CPU profile of the process. + +The workload is shaped by what the dashboards need rather than by what is +convenient. Every lane runs, so per-transport panels have more than one series. +Grouped commands run, so the inlining panels are not flat zero. Commands that +tmux rejects run on purpose, so the failure panels have data -- a dashboard +whose error widget is empty is untested, not healthy. + +Run it through ``just otel-smoke`` rather than directly; the recipe supplies the +endpoints. +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import os +import pathlib +import shutil +import subprocess +import sys +import time +import typing as t +import uuid + +sys.path.insert(0, str(pathlib.Path(__file__).parent / "lgtm")) + +import telemetry + +from libtmux.experimental.engines import ( + AsyncControlModeEngine, + AsyncSubprocessEngine, + ControlModeEngine, + SubprocessEngine, + instrument, +) +from libtmux.experimental.engines.base import ( + CommandRequest, + CommandSeparator, +) +from libtmux.experimental.engines.instrumentation import CountingSink +from libtmux.server import Server + +logger = logging.getLogger("libtmux.otel_smoke") + +PLAIN = CommandRequest.from_args("list-panes", "-a", "-F", "#{pane_id}") +LISTING = CommandRequest.from_args("list-windows", "-a", "-F", "#{window_id}") +GROUPED = CommandRequest.from_args( + "set-option", + "-g", + "@smoke", + "1", + CommandSeparator(";"), + "show-options", + "-g", + "@smoke", +) +# tmux rejects this: the target window does not exist. It is here so the +# failure counters and the error-rate panels receive real data. +REJECTED = CommandRequest.from_args("list-panes", "-t", "@999999") + +CYCLE = (PLAIN, GROUPED, LISTING, PLAIN, GROUPED, REJECTED) + + +def start_server(root: pathlib.Path) -> pathlib.Path: + """Create a throwaway tmux server and return its socket path.""" + socket_path = root / "smoke.sock" + subprocess.run( + ( + "tmux", + "-S", + str(socket_path), + "-f", + "/dev/null", + "new-session", + "-d", + "-s", + "smoke", + "sleep 300", + ), + check=True, + ) + # Control mode opens a persistent client only when detaching is safe. + subprocess.run( + ( + "tmux", + "-S", + str(socket_path), + "set-option", + "-g", + "destroy-unattached", + "off", + ), + check=True, + ) + for index in range(3): + subprocess.run( + ( + "tmux", + "-S", + str(socket_path), + "new-window", + "-t", + "smoke", + "-n", + f"w{index}", + "sleep 300", + ), + check=True, + ) + return socket_path + + +def run_sync(engine: t.Any, seconds: float) -> None: + """Issue the command cycle until the deadline passes.""" + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + for request in CYCLE: + engine.run(request) + + +async def run_async(engine: t.Any, seconds: float, concurrency: int) -> None: + """Issue the cycle from several tasks at once. + + Overlapping tasks are the point: they exercise the async wrappers under the + concurrency the engines are built for, and they make the span timeline show + real overlap rather than a single serial chain. + """ + deadline = time.monotonic() + seconds + + async def worker() -> None: + while time.monotonic() < deadline: + for request in CYCLE: + await engine.run(request) + + await asyncio.gather(*(worker() for _ in range(concurrency))) + + +def lane_totals(counts: CountingSink) -> dict[str, int]: + """Summarize one lane's locally observed counts.""" + return { + "requests": counts.requests, + "tmux_commands": counts.tmux_commands, + "inlined": counts.inlined, + "elapsed_ms": round(counts.elapsed_ns / 1e6), + } + + +def main(argv: list[str] | None = None) -> int: + """Run every lane under full telemetry and print the local counts.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-id", default=f"smoke-{uuid.uuid4().hex[:8]}") + parser.add_argument( + "--seconds", type=float, default=4.0, help="workload duration per lane" + ) + parser.add_argument( + "--concurrency", + type=int, + default=4, + help="overlapping tasks in the async lanes", + ) + parser.add_argument( + "--otlp", + default=os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:4318"), + ) + parser.add_argument( + "--pyroscope", + default=os.environ.get("PYROSCOPE_SERVER_ADDRESS", "http://127.0.0.1:4040"), + ) + args = parser.parse_args(argv) + + os.environ.pop("TMUX", None) + os.environ.pop("TMUX_PANE", None) + + signals = telemetry.build(args.otlp, run_id=args.run_id) + logging.basicConfig(level=logging.INFO, handlers=[signals.handler], force=True) + + try: + import pyroscope + + pyroscope.configure( + application_name=telemetry.SERVICE_NAME, + server_address=args.pyroscope, + sample_rate=100, + upload_interval=3, + tags={"run_id": args.run_id}, + ) + profiling = True + except Exception as error: # noqa: BLE001 - profiling is optional + logger.warning("profiling disabled: %s", error) + profiling = False + + root = pathlib.Path(f"/tmp/libtmux-smoke-{uuid.uuid4().hex[:8]}") + root.mkdir(mode=0o700) + totals: dict[str, dict[str, int]] = {} + try: + socket_path = start_server(root) + server = Server(socket_path=socket_path, config_file=os.devnull) + logger.info("smoke run started", extra={"run_id": args.run_id, "lanes": 4}) + + sync_lanes = ( + ("subprocess", lambda: SubprocessEngine.for_server(server)), + ("control", lambda: ControlModeEngine.for_server(server)), + ) + for lane, factory in sync_lanes: + counts = CountingSink() + engine = instrument( + factory(), + counts, + telemetry.OTelSink(signals.tracer, signals.meter, lane), + ) + run_sync(engine, args.seconds) + totals[lane] = lane_totals(counts) + logger.info("lane finished", extra={"lane": lane, **totals[lane]}) + + async_lanes = ( + ("subprocess-async", lambda: AsyncSubprocessEngine.for_server(server)), + ("control-async", lambda: AsyncControlModeEngine.for_server(server)), + ) + for lane, factory in async_lanes: + counts = CountingSink() + engine = instrument( + factory(), + counts, + telemetry.OTelSink(signals.tracer, signals.meter, lane), + ) + + async def drive(engine: t.Any = engine) -> None: + try: + await run_async(engine, args.seconds, args.concurrency) + finally: + inner = engine.inner + if hasattr(inner, "aclose"): + await inner.aclose() + + asyncio.run(drive()) + totals[lane] = lane_totals(counts) + logger.info("lane finished", extra={"lane": lane, **totals[lane]}) + finally: + subprocess.run( + ("tmux", "-S", str(root / "smoke.sock"), "kill-server"), + capture_output=True, + check=False, + ) + shutil.rmtree(root, ignore_errors=True) + + if profiling: + # Pyroscope batches on an interval; give it one before shutting down. + time.sleep(4) + import pyroscope + + pyroscope.shutdown() + signals.shutdown() + + header = ( + f"{'lane':<18}{'requests':>10}{'tmuxcmd':>10}{'inlined':>10}{'engine_ms':>11}" + ) + print(f"\n {header}") + print(" " + "-" * len(header)) + for lane, row in totals.items(): + print( + f" {lane:<18}{row['requests']:>10}{row['tmux_commands']:>10}" + f"{row['inlined']:>10}{row['elapsed_ms']:>11}" + ) + print(f"\n run_id={args.run_id} service={telemetry.SERVICE_NAME}") + print(f" exported to {args.otlp} and {args.pyroscope}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_lgtm_dashboards.py b/tests/test_lgtm_dashboards.py new file mode 100644 index 000000000..7912f714d --- /dev/null +++ b/tests/test_lgtm_dashboards.py @@ -0,0 +1,179 @@ +"""Structural contracts for the generated Grafana dashboards. + +These run offline. They cannot tell whether a panel has data -- that is +``scripts/otel_acceptance.py`` against a live stack -- but they do keep the +committed JSON honest about its generator and about the datasources it binds +to, which is where a board rots silently. +""" + +from __future__ import annotations + +import importlib.util +import json +import pathlib +import typing as t + +import pytest + +_ROOT = pathlib.Path(__file__).parents[1] +_LGTM = _ROOT / "scripts" / "lgtm" +_DASHBOARDS = _LGTM / "dashboards" + +# The uids provisioned by scripts/lgtm/grafana-datasources.yaml. A panel bound +# to anything else renders an error instead of data. +_DATASOURCE_UIDS = {"prometheus", "loki", "tempo", "pyroscope"} + + +def _generator() -> t.Any: + """Import the dashboard generator from ``scripts/`` by path.""" + spec = importlib.util.spec_from_file_location( + "generate_dashboards", _LGTM / "generate_dashboards.py" + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _boards() -> list[dict[str, t.Any]]: + """Load every committed dashboard.""" + loaded: list[dict[str, t.Any]] = [ + json.loads(path.read_text(encoding="utf-8")) + for path in sorted(_DASHBOARDS.glob("*.json")) + ] + assert loaded, "no dashboards committed" + return loaded + + +def _panels(board: dict[str, t.Any]) -> list[dict[str, t.Any]]: + """Return a board's real panels, skipping row headers.""" + return [panel for panel in board["panels"] if panel["type"] != "row"] + + +def test_committed_dashboards_match_their_generator(tmp_path: pathlib.Path) -> None: + """The JSON in the repo is what the generator produces right now. + + ``up.sh`` regenerates on every start, so a stale committed copy would be + silently replaced at runtime and the diff would show up in someone else's + working tree. + """ + module = _generator() + module.write_dashboards(tmp_path) + + regenerated = { + path.name: path.read_text(encoding="utf-8") for path in tmp_path.glob("*.json") + } + committed = { + path.name: path.read_text(encoding="utf-8") + for path in _DASHBOARDS.glob("*.json") + } + + assert set(regenerated) == set(committed) + for name, text in regenerated.items(): + assert committed[name] == text, f"{name} is stale; run `just otel-dashboards`" + + +def test_every_panel_queries_something() -> None: + """A panel with no target can never show data.""" + for board in _boards(): + for panel in _panels(board): + assert panel.get("targets"), ( + f"{board['uid']}/{panel['title']} has no targets" + ) + + +def test_every_target_binds_a_provisioned_datasource() -> None: + """Panels reference datasources by uid, so the uid has to exist.""" + for board in _boards(): + for panel in _panels(board): + for target in panel["targets"]: + uid = (target.get("datasource") or {}).get("uid") + assert uid in _DATASOURCE_UIDS, ( + f"{board['uid']}/{panel['title']} binds unknown datasource {uid!r}" + ) + + +def test_lane_variable_is_defined_wherever_it_is_used() -> None: + """A query filtering on ``$lane`` needs the board to define ``$lane``. + + An undefined variable is not an error in Grafana; it interpolates to an + empty string and the panel quietly returns nothing. + """ + for board in _boards(): + names = {var["name"] for var in board["templating"]["list"]} + serialized = json.dumps(board["panels"]) + if "$lane" in serialized: + assert "lane" in names, f"{board['uid']} uses $lane without defining it" + + +def test_dashboard_uid_matches_its_filename() -> None: + """Provisioning keys on uid; a mismatch makes boards hard to find.""" + for path in sorted(_DASHBOARDS.glob("*.json")): + board = json.loads(path.read_text(encoding="utf-8")) + assert board["uid"] == path.stem + + +def test_acceptance_expands_every_template_variable() -> None: + """No ``$`` may survive expansion, or a query filters on a literal. + + Grafana substitutes variables before sending a query. The acceptance script + has to do the same, and a variable it does not know about would be sent + through as text and match nothing. + """ + spec = importlib.util.spec_from_file_location( + "otel_acceptance", _ROOT / "scripts" / "otel_acceptance.py" + ) + assert spec is not None + assert spec.loader is not None + acceptance = importlib.util.module_from_spec(spec) + spec.loader.exec_module(acceptance) + + for board in _boards(): + for panel in _panels(board): + for target in panel["targets"]: + query = ( + target.get("expr") + or target.get("query") + or target.get("labelSelector") + ) + if query is None: + continue + assert "$" not in acceptance.expand(query), ( + f"{board['uid']}/{panel['title']} keeps a variable after expansion" + ) + + +@pytest.mark.parametrize( + "required", ["libtmux-overview", "libtmux-transports", "libtmux-commands"] +) +def test_expected_boards_are_committed(required: str) -> None: + """The suite the README and acceptance script assume exists.""" + assert (_DASHBOARDS / f"{required}.json").is_file() + + +def test_no_panel_reads_a_counter_without_a_window() -> None: + """Every Prometheus query must span a range, never read a counter at a point. + + The workloads that feed these boards are short-lived. A few minutes after + one exits, Prometheus marks its series stale and an instant query at + ``now`` returns nothing, so a panel reading ``sum(tmux_requests_total)`` + renders "No data" while the timeseries beside it still shows the run. + + Windowing the query -- ``increase(...[$__range])``, ``rate(...[...])`` -- + asks what happened during the selected window instead, which is both what + the viewer meant and immune to staleness. This test is the cheap offline + guard for that: a live check only catches it if it runs late enough for + the series to have gone stale, which is exactly when nobody is looking. + """ + for board in _boards(): + for panel in _panels(board): + for target in panel["targets"]: + if (target.get("datasource") or {}).get("type") != "prometheus": + continue + expr = target["expr"] + assert "[" in expr, ( + f"{board['uid']}/{panel['title']} reads a counter with no " + f"range window and will blank out once the series goes " + f"stale: {expr}" + ) diff --git a/uv.lock b/uv.lock index 652703f50..ac50bb7c6 100644 --- a/uv.lock +++ b/uv.lock @@ -765,6 +765,18 @@ server = [ { name = "websockets" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.75.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/73/74bcab964c9a7a61f2bb71e8179b0f13e6fa98f7ce00fd168aab291e4a2e/googleapis_common_protos-1.75.1.tar.gz", hash = "sha256:d3042c6c5a2d4e67113104d6b6818b59b6bd92a197f2a91508e801fe815cf071", size = 150967, upload-time = "2026-08-06T06:24:51.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/51/186c02b8549b69ccda44429cf6ff5081e4b61a602ddfe6a8020d1be31d1b/googleapis_common_protos-1.75.1-py3-none-any.whl", hash = "sha256:28a1934bcd33b9c9da66ac301a0a4227e3367f095a17d0375cb98f0a09d93b79", size = 300626, upload-time = "2026-08-06T06:23:46.696Z" }, +] + [[package]] name = "gp-furo-theme" version = "0.1.0a37" @@ -1199,6 +1211,11 @@ lint = [ { name = "ruff" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] +otel = [ + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, + { name = "pyroscope-io" }, +] testing = [ { name = "fastmcp" }, { name = "gp-libs" }, @@ -1256,6 +1273,11 @@ lint = [ { name = "ruff", specifier = ">=0.16.1" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] +otel = [ + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, + { name = "pyroscope-io" }, +] testing = [ { name = "fastmcp" }, { name = "gp-libs", specifier = ">=0.0.19" }, @@ -1581,14 +1603,83 @@ wheels = [ [[package]] name = "opentelemetry-api" -version = "1.43.0" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.44.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ae/cc/e4c9584181f86494df0f6bdec1a4f3280c50db44704dc2a407e994fc87bb/opentelemetry_api-1.43.0.tar.gz", hash = "sha256:107d0d03857ea8fc7c5fcbbbd83f800c281f0d560553d61c1d675fccfd1761c1", size = 73476, upload-time = "2026-06-24T15:19:55.323Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/83/6dba32b85f31868400440dc7ad2ca1eab94cbbf3a7b0459ed39f8311a9e2/opentelemetry_api-1.43.0-py3-none-any.whl", hash = "sha256:20acf45e9b21851926835292e4045d290acade1edd2ff3de86d2f069687ba1fd", size = 61912, upload-time = "2026-06-24T15:19:35.434Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.65b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, ] [[package]] @@ -1636,6 +1727,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "protobuf" +version = "7.35.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" }, + { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" }, + { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, +] + [[package]] name = "py-key-value-aio" version = "0.4.5" @@ -1856,6 +1962,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/80/fc9d01d5ed37ba4c42ca2b55b4339ae6e200b456be3a1aaddf4a9fa99b8c/pyperclip-1.11.0-py3-none-any.whl", hash = "sha256:299403e9ff44581cb9ba2ffeed69c7aa96a008622ad0c46cb575ca75b5b84273", size = 11063, upload-time = "2025-09-26T14:40:36.069Z" }, ] +[[package]] +name = "pyroscope-io" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/d2/5fc44302f861eb2fd19bf7e56423c93e8867199c647337bb9849bc6d2929/pyroscope_io-1.2.1.tar.gz", hash = "sha256:c3236136dc086845d283fbeb434f5e22f5a7ddc0ff5a5b328752335d61ee1aaf", size = 74584, upload-time = "2026-07-27T11:39:44.57Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/bb/0095807d6be83b79a18e06668ae3289da0a05611efb880617e9dc0fde462/pyroscope_io-1.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e2ddf9ddbecea6625fce29c1abd4add9dbf17ae2d0cb4dece05110de91f1cddb", size = 2115347, upload-time = "2026-07-27T11:39:00.29Z" }, + { url = "https://files.pythonhosted.org/packages/68/1b/20ca1dd1bf7d09bb359d05fae7b88fd668e8ae953293b9108d041cd39299/pyroscope_io-1.2.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:acdc935c60e5f84917e47b14a6fe14a8dbb93735b89e0b1a85eab5c27e4be134", size = 2194622, upload-time = "2026-07-27T11:39:01.825Z" }, + { url = "https://files.pythonhosted.org/packages/06/53/8520a21e5f033d39cfc050ce751acbdcf4cd48be0206a7b9eae80d5e4dde/pyroscope_io-1.2.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:71278a8ca7babd5b21dfb8ff7a0ba37c35a82e754f42bcceb96c8a0e1b7f44f4", size = 5584022, upload-time = "2026-07-27T11:39:03.29Z" }, + { url = "https://files.pythonhosted.org/packages/eb/26/e9faea018ce68aa5a18a425fe21a219b7f6dd047795dcf9a55b1e2418d0c/pyroscope_io-1.2.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b2b2c6833c95881208222a8675bf0d8a90a1225748a185e73811b956f055b45f", size = 5127220, upload-time = "2026-07-27T11:39:04.991Z" }, + { url = "https://files.pythonhosted.org/packages/65/a7/63fdb56e1943bf1a51f0a93dd485994d3fbc2871ea8b976531b1db4ed367/pyroscope_io-1.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd7e1963ba0075ab8f8dd2877e059e1c02d9ba903f55696541b3cdb3b9df5378", size = 5513990, upload-time = "2026-07-27T11:39:06.467Z" }, + { url = "https://files.pythonhosted.org/packages/23/f0/475ed4b480e554359e29c5dd5f28e251e49d9ff6e009618412ced5d0ab02/pyroscope_io-1.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d5f59a62785c1ce56247bfacf1e3f80c2af14df227585bc738dc6d2baab7c315", size = 5244157, upload-time = "2026-07-27T11:39:07.952Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f3/0b8c4906c6622c1c1451b52cacc192db41e197d3c6db60b235909e558c94/pyroscope_io-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8387470016135a590f43da21a4dfcd47c238e4dfacabccc3e527da6cf61898f7", size = 2116130, upload-time = "2026-07-27T11:39:09.309Z" }, + { url = "https://files.pythonhosted.org/packages/3f/5c/c925b3a127071d0d06cffb0152db9c452f375d711d77b942f08e77d27d37/pyroscope_io-1.2.1-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:0c8eb803eed4ae7247adb18999afcfb82894c28d67c0b99c63c16fc6443c63cc", size = 2195008, upload-time = "2026-07-27T11:39:10.638Z" }, + { url = "https://files.pythonhosted.org/packages/85/56/832ab29d946c5b94a1ced1f4547964318c2dc82bb06c8b8501c883dfce2d/pyroscope_io-1.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bebcccb717abddeb59b7df12e693fd0f85b946a20a38bdcfef3491fbac7d827c", size = 5583871, upload-time = "2026-07-27T11:39:12.278Z" }, + { url = "https://files.pythonhosted.org/packages/7b/7d/9ff9223852965390fb17a6548f129fd73e038584d8d4b5c3e65ba83d8815/pyroscope_io-1.2.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4327785c6087afc084a230aa27107b53070c67555438caf3fce3ac73f821c787", size = 5127972, upload-time = "2026-07-27T11:39:13.636Z" }, + { url = "https://files.pythonhosted.org/packages/60/e6/fa8917594d1a3cb06e4400b49f95a4760e59e890de48360f9bcc29ca26e9/pyroscope_io-1.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d18f843ef60503f15c9d189dc1aa7f06d8edac9760f252ae6e825805fc7d21e3", size = 5513939, upload-time = "2026-07-27T11:39:15.078Z" }, + { url = "https://files.pythonhosted.org/packages/b7/ab/d9b6d65f954e6d0d9f95e8468ec61b526c4e2aa3c0332d5b609cd89ad6eb/pyroscope_io-1.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:736d569a465edafc5c271d702bfef24e9afd25c06e5ea2e6746bd01ef622e62e", size = 5244667, upload-time = "2026-07-27T11:39:16.554Z" }, + { url = "https://files.pythonhosted.org/packages/1a/2c/cfaddac31c80b471cc896d3d09ee88d99779861a6b6dc3a7f7d43f12e39e/pyroscope_io-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:692091ae7d020ee76a16d9da2e213b69c20367d20032ecb643c23e3a5831f87d", size = 2113014, upload-time = "2026-07-27T11:39:18.237Z" }, + { url = "https://files.pythonhosted.org/packages/5b/d5/188f433e9ab08973948fac72e6aacf2a641a69c45885bed982cc8a768e64/pyroscope_io-1.2.1-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:ceabd42b2d61876eb81668cb76bb04b911d62369645c1c4eb63c9d4ed5106b3c", size = 2196053, upload-time = "2026-07-27T11:39:19.594Z" }, + { url = "https://files.pythonhosted.org/packages/96/86/fa0bca1756f881b9960c460cef7604e9044bb84a68d9d76cbd3e39372cf3/pyroscope_io-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e64e2f34f340b163013b2942017d98bb0a4584e26ae64dfdba3f85304eea8efe", size = 5579568, upload-time = "2026-07-27T11:39:20.804Z" }, + { url = "https://files.pythonhosted.org/packages/de/1c/a7dac80341fe67ae775a33f9d0da2c04445479be3ce8a96e4b62df210c6e/pyroscope_io-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:16a39b80e5032fe978eaac6abf907e7b79128357adc0f0d8e7921e68bd57c4ea", size = 5123015, upload-time = "2026-07-27T11:39:22.185Z" }, + { url = "https://files.pythonhosted.org/packages/6d/90/049e9fda943f6fd01f035783c633d3e83a1a34aa00b1ef1eb325f0b0aa86/pyroscope_io-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:45f5d7ed4a7a158732d3c1634846011c382309720d891200f64aee3a11173418", size = 5510537, upload-time = "2026-07-27T11:39:23.633Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9b/f35ccd7935f87aa67a60e158b45f992e2e08243c5c6b432567cf7b1ef194/pyroscope_io-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7bc01f8b5c49efca59987194c28667559b1fbded965943d020d12aadc40d0c5c", size = 5239529, upload-time = "2026-07-27T11:39:25.532Z" }, + { url = "https://files.pythonhosted.org/packages/fc/11/8a3d16443d157f817975e61f2e97e894a6e2966cfe700cb900a4f16caf9e/pyroscope_io-1.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e2ce28485fc46390c31300abde8c93b7dd6c042960deb1b729c2d001a8795516", size = 2112233, upload-time = "2026-07-27T11:39:26.889Z" }, + { url = "https://files.pythonhosted.org/packages/cd/34/640bcbeb50aec8dac27b4242b349a2a2eeb3c4ab8b62660ae9cb2dbe1cee/pyroscope_io-1.2.1-cp313-cp313-macosx_11_0_x86_64.whl", hash = "sha256:10b0e7a8040ed6b953c9920f2c463d6dddd4f26c01dcb1ffb696879c15cab218", size = 2195244, upload-time = "2026-07-27T11:39:28.397Z" }, + { url = "https://files.pythonhosted.org/packages/03/07/d3e7c279d44b88e94484453574ff837b0fecae3395a979d4def25b548234/pyroscope_io-1.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:655f8629f3f8c5b8c2bec105f0c19a69be4666ae5e452e4c910b2fa3de3452d6", size = 5578633, upload-time = "2026-07-27T11:39:29.805Z" }, + { url = "https://files.pythonhosted.org/packages/4d/59/9ce4cdafc9a31eda7aeb37c931367c61dab7846b56e396259273efa46e6f/pyroscope_io-1.2.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e232ea23cc756f6990a325230c6f22d7c380b012a3bd9dac476a338b1edd7d26", size = 5121946, upload-time = "2026-07-27T11:39:31.478Z" }, + { url = "https://files.pythonhosted.org/packages/19/80/cc8eb25b908e27fc7ef63b007eb226e3c31903fd9886ab18dea1fccda2c6/pyroscope_io-1.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:35e57ab13d40d8af15821bc30720eb5c74949c169b649379ad114f7e1f75cd8f", size = 5510621, upload-time = "2026-07-27T11:39:32.939Z" }, + { url = "https://files.pythonhosted.org/packages/ad/9c/6452ac8356c4f7f11e67a96bc57062b813158e1315fad63ac81008d5ed79/pyroscope_io-1.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:31a2a694f507280a451fe082789599e68af48389c74b2c15d99715bd0d0f9f77", size = 5238044, upload-time = "2026-07-27T11:39:34.654Z" }, + { url = "https://files.pythonhosted.org/packages/3e/ec/dd4911e8a36e40050b14f382489de9bac474e2c02cefaa5e02e04a59edd3/pyroscope_io-1.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:152be5e004ca87da17a91c9b80726b78b32d3d4ae3ed0e31117924f76c665c7a", size = 2113631, upload-time = "2026-07-27T11:39:35.953Z" }, + { url = "https://files.pythonhosted.org/packages/a3/6a/4c4067e7a9cf67329f431db30dffff00fd0cdd4fff013bfa7801ce8eacd0/pyroscope_io-1.2.1-cp314-cp314-macosx_11_0_x86_64.whl", hash = "sha256:95d535c377c43631b1a4d24ac42f3fa81ece7baca4749f23a6cc9db5ff98e477", size = 2196267, upload-time = "2026-07-27T11:39:37.448Z" }, + { url = "https://files.pythonhosted.org/packages/04/b3/571e0065c8e3b54f36ddbdc9faa79db56647c1bc8d7f0fb28cddcef096ba/pyroscope_io-1.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8673378adcf7985b6824cefbdfc8ee2567576d1e70f710b8dda649cb3a90b1ae", size = 5579251, upload-time = "2026-07-27T11:39:38.747Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7e/908489444509ec91ba0f2d165ba745a0315dcf8eb9a17f55af3b17bec708/pyroscope_io-1.2.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ec8e2a8954de298000811af743060c2ec85295c452a8b3e0226be11ffaf1c80", size = 5123892, upload-time = "2026-07-27T11:39:40.104Z" }, + { url = "https://files.pythonhosted.org/packages/f9/17/aeaeb24938460348c9aa7903cfc186d6c1130b0d097fbddc5c9db0dc517f/pyroscope_io-1.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:014c6f222cec75eebb12b85ea8ced49efedd29f027dae915e4257822b2815272", size = 5510572, upload-time = "2026-07-27T11:39:41.575Z" }, + { url = "https://files.pythonhosted.org/packages/e2/07/92941723d8aaca1dbbe0f724da51ff8d02e610353b5e999de97a0547f950/pyroscope_io-1.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:83867ba5f46f1d0946524407d5627c1bb06f32dfd35fc06c7258f756d9e80c08", size = 5240581, upload-time = "2026-07-27T11:39:43.217Z" }, +] + [[package]] name = "pytest" version = "9.1.1" From f116d1da764640ef81d65d6f836cd2702a059539 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 08:28:34 -0500 Subject: [PATCH 03/24] Bench(feat[lgtm]): Stamp runs with branch, worktree, and spike identity why: Telemetry that cannot say which branch, worktree, or experiment produced it can be observed but not compared, which is most of what a benchmark is for. The trap is treating that as one decision: copying every fact onto every signal makes each commit mint a fresh set of Prometheus series, and the cost lands on whoever runs the stack next month. what: - Add scripts/lgtm/identity.py resolving repository, ref, revision, and worktree from one git rev-parse at startup, using the OpenTelemetry vcs.* conventions, overridable by environment for CI's detached checkouts - Split the facts by signal: metrics carry only branch, run id, and spike, the dimensions worth grouping by; revision and worktree ride on traces and profiles where the drill-down happens. A SHA is never a comparison axis -- each run has one, so grouping by it is grouping by run - Copy baggage onto spans with a processor, so a value that changes mid-run reaches spans created inside the engines without becoming a parameter on calls that have no business knowing about telemetry - Give every board Transport and Branch selectors, and add a Compare board grouping the same measurements by run and by branch - Add tests pinning the metric label set closed and rejecting a panel that filters on an undefined variable --- CHANGES | 9 + scripts/lgtm/README.md | 63 +- scripts/lgtm/dashboards/libtmux-commands.json | 37 +- scripts/lgtm/dashboards/libtmux-compare.json | 562 ++++++++++++++++++ scripts/lgtm/dashboards/libtmux-overview.json | 39 +- .../lgtm/dashboards/libtmux-transports.json | 41 +- scripts/lgtm/generate_dashboards.py | 187 +++++- scripts/lgtm/identity.py | 245 ++++++++ scripts/lgtm/telemetry.py | 106 +++- scripts/otel_acceptance.py | 5 + scripts/otel_smoke.py | 30 +- tests/test_lgtm_dashboards.py | 54 ++ 12 files changed, 1326 insertions(+), 52 deletions(-) create mode 100644 scripts/lgtm/dashboards/libtmux-compare.json create mode 100644 scripts/lgtm/identity.py diff --git a/CHANGES b/CHANGES index 26125b4a9..1713844bf 100644 --- a/CHANGES +++ b/CHANGES @@ -289,6 +289,15 @@ The Grafana dashboards under `scripts/lgtm/dashboards/` are generated rather than hand-edited, and `scripts/otel_acceptance.py` runs each panel's own query and fails naming any panel that came back empty. See `scripts/lgtm/README.md`. +Every run is stamped with its branch, revision, repository, worktree, and an +optional spike name, so two runs can be compared rather than merely observed. +Which signal carries which fact is a deliberate split: metrics take only the +dimensions worth grouping by, while the revision, worktree, and per-test +identity ride on traces and profiles where high cardinality is expected. +Values that change within a process, such as the phase a workload is in, travel +as OpenTelemetry baggage and are copied onto spans by a processor instead of +being threaded through engine calls. + #### CI actions updated to current majors Workflow actions moved to their current major releases: `actions/checkout` v7, diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md index 0cc6b5ffe..88646d170 100644 --- a/scripts/lgtm/README.md +++ b/scripts/lgtm/README.md @@ -89,14 +89,73 @@ The workload deliberately issues commands tmux rejects. A dashboard whose error panel is empty is untested rather than healthy, so the failure path has to produce real data. +## Identity: which fact rides on which signal + +Every run is stamped with where it came from -- branch, revision, repository, +worktree, and optionally a spike name -- so two runs can be told apart and +compared. The interesting decision is not collecting that, it is choosing which +signal carries which fact, because the three fail differently when you get it +wrong. + +| Fact | Metrics | Traces | Profiles | +| ---- | ------- | ------ | -------- | +| branch (`vcs.ref.head.name`) | yes | yes | yes | +| run id (`libtmux.run_id`) | yes | yes | yes | +| spike (`libtmux.spike`) | yes | yes | yes | +| revision (`vcs.ref.head.revision`) | **no** | yes | yes | +| worktree (`libtmux.worktree`) | **no** | yes | yes | +| test case, phase | **no** | yes | no | + +A metric label is a stored time series forever, so the metric row is a budget +rather than a wish list. The test is "would I group by this?", not "is it +interesting?". A commit SHA fails that test: each run has exactly one, so +grouping by SHA is grouping by run, which the run id already does -- it would +add no query power while creating a fresh set of series on every commit. It +still rides on spans and profiles, where the drill-down happens and high +cardinality is expected. `scripts/lgtm/identity.py` owns the split, and a test +fails if the metric set grows. + +Resolving all of it costs one `git rev-parse` at process start, a couple of +milliseconds, and nothing per tmux command: the labels are computed once and +merged into each point. `LIBTMUX_VCS_REF`, `LIBTMUX_VCS_REVISION`, +`LIBTMUX_WORKTREE`, and `LIBTMUX_SPIKE` override the detected values, which is +what CI wants -- it checks out a detached HEAD but knows the branch the work +belongs to. + +### Baggage, for what changes mid-process + +Branch and revision are fixed for a process, so they are resource attributes. +The test now running, or the phase a workload is in, are not -- and they should +not become parameters threaded through engine calls that have no business +knowing about telemetry. + +`telemetry.scope()` puts them in OpenTelemetry baggage, and a span processor +copies the approved keys onto every span as it starts: + +```python +with telemetry.scope(**{"libtmux.phase": "control-async"}): + ... +``` + +Spans created anywhere inside that block carry `libtmux.phase`, queryable in +Tempo as `{ span.libtmux.phase = "control-async" }`. Only the keys in +`identity.BAGGAGE_KEYS` are copied, because baggage propagates across process +boundaries and an unrelated caller's entries should not silently become +attributes here. The cost is one context read per span, and when no baggage is +set the loop body never runs. + ## Dashboards -Three boards, provisioned into the `libtmux` folder: +Four boards, provisioned into the `libtmux` folder: `libtmux / Overview` is throughput, latency, failures, and the trace, log, and profile panels side by side. `libtmux / Transports` compares the four transports against each other. `libtmux / Commands` breaks the same work down by tmux -command. +command. `libtmux / Compare` answers "is this different from that", grouping the +same measurements by run and by branch. + +Every board carries Transport and Branch selectors, and Compare adds a Spike +selector to scope a comparison to one experiment. The JSON is generated, not hand-written, by `generate_dashboards.py`. A board is a few hundred lines of nested objects where panel placement is manual diff --git a/scripts/lgtm/dashboards/libtmux-commands.json b/scripts/lgtm/dashboards/libtmux-commands.json index a077662bc..dff961a8b 100644 --- a/scripts/lgtm/dashboards/libtmux-commands.json +++ b/scripts/lgtm/dashboards/libtmux-commands.json @@ -40,6 +40,29 @@ "text": "All", "value": "$__all" } + }, + { + "name": "branch", + "label": "Branch", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(tmux_requests_total, vcs_ref_head_name)", + "refId": "var-branch" + }, + "refresh": 2, + "sort": 1, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } } ] }, @@ -88,7 +111,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_command) (increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range]))", + "expr": "sum by (tmux_command) (increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range]))", "legendFormat": "{{tmux_command}}", "range": false, "instant": true, @@ -147,7 +170,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_command) (rate(tmux_requests_total{tmux_lane=~\"$lane\"}[$__rate_interval]))", + "expr": "sum by (tmux_command) (rate(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__rate_interval]))", "legendFormat": "{{tmux_command}}", "range": true, "instant": false, @@ -222,7 +245,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le, tmux_command) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le, tmux_command) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__rate_interval])))", "legendFormat": "{{tmux_command}}", "range": true, "instant": false, @@ -280,7 +303,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_command) (rate(tmux_failures_total{tmux_lane=~\"$lane\"}[$__rate_interval]))", + "expr": "sum by (tmux_command) (rate(tmux_failures_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__rate_interval]))", "legendFormat": "{{tmux_command}}", "range": true, "instant": false, @@ -338,7 +361,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_command) (increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range]))", + "expr": "sum by (tmux_command) (increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range]))", "legendFormat": "requests", "range": false, "instant": true, @@ -351,7 +374,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_command) (increase(tmux_commands_total{tmux_lane=~\"$lane\"}[$__range]))", + "expr": "sum by (tmux_command) (increase(tmux_commands_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range]))", "legendFormat": "commands", "range": false, "instant": true, @@ -364,7 +387,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le, tmux_command) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__range])))", + "expr": "histogram_quantile(0.95, sum by (le, tmux_command) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range])))", "legendFormat": "p95", "range": false, "instant": true, diff --git a/scripts/lgtm/dashboards/libtmux-compare.json b/scripts/lgtm/dashboards/libtmux-compare.json new file mode 100644 index 000000000..eb5746d47 --- /dev/null +++ b/scripts/lgtm/dashboards/libtmux-compare.json @@ -0,0 +1,562 @@ +{ + "uid": "libtmux-compare", + "title": "libtmux / Compare", + "description": "One run against another, one branch against another. Pick a spike to scope the comparison to a single experiment.", + "tags": [ + "libtmux", + "generated" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "lane", + "label": "Transport", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(tmux_requests_total, tmux_lane)", + "refId": "var-lane" + }, + "refresh": 2, + "sort": 1, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + }, + { + "name": "branch", + "label": "Branch", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(tmux_requests_total, vcs_ref_head_name)", + "refId": "var-branch" + }, + "refresh": 2, + "sort": 1, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + }, + { + "name": "spike", + "label": "Spike", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(tmux_requests_total, libtmux_spike)", + "refId": "var-spike" + }, + "refresh": 2, + "sort": 1, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "links": [ + { + "title": "libtmux dashboards", + "type": "dashboards", + "tags": [ + "libtmux" + ], + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "icon": "external link" + } + ], + "annotations": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "By run", + "collapsed": false, + "id": 1, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "table", + "title": "Runs in range", + "description": "Every run in the window, side by side.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (libtmux_run_id) (increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\", libtmux_spike=~\"$spike\"}[$__range]))", + "legendFormat": "requests", + "range": false, + "instant": true, + "refId": "A", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (libtmux_run_id) (increase(tmux_commands_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\", libtmux_spike=~\"$spike\"}[$__range]))", + "legendFormat": "commands", + "range": false, + "instant": true, + "refId": "B", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (libtmux_run_id) (increase(tmux_failures_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\", libtmux_spike=~\"$spike\"}[$__range]))", + "legendFormat": "failures", + "range": false, + "instant": true, + "refId": "C", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, libtmux_run_id) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\", libtmux_spike=~\"$spike\"}[$__range])))", + "legendFormat": "p95", + "range": false, + "instant": true, + "refId": "D", + "format": "table" + } + ], + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true, + "instance": true + } + } + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto" + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "footer": { + "show": false + } + }, + "id": 2, + "gridPos": { + "x": 0, + "y": 1, + "w": 24, + "h": 9 + } + }, + { + "type": "timeseries", + "title": "p95 by run", + "description": "Did a run get slower than the one before it?", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, libtmux_run_id) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\", libtmux_spike=~\"$spike\"}[$__range])))", + "legendFormat": "{{libtmux_run_id}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "s", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 3, + "gridPos": { + "x": 0, + "y": 10, + "w": 12, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "Request rate by run", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (libtmux_run_id) (rate(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\", libtmux_spike=~\"$spike\"}[$__rate_interval]))", + "legendFormat": "{{libtmux_run_id}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 4, + "gridPos": { + "x": 12, + "y": 10, + "w": 12, + "h": 8 + } + }, + { + "type": "row", + "title": "By branch", + "collapsed": false, + "id": 5, + "gridPos": { + "x": 0, + "y": 18, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "p95 by branch", + "description": "The regression check: one branch against another.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, vcs_ref_head_name) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\", libtmux_spike=~\"$spike\"}[$__range])))", + "legendFormat": "{{vcs_ref_head_name}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "s", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "lineWidth": 2 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 6, + "gridPos": { + "x": 0, + "y": 19, + "w": 12, + "h": 8 + } + }, + { + "type": "piechart", + "title": "Requests by branch", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (vcs_ref_head_name) (increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\", libtmux_spike=~\"$spike\"}[$__range]))", + "legendFormat": "{{vcs_ref_head_name}}", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "id": 7, + "gridPos": { + "x": 12, + "y": 19, + "w": 12, + "h": 8 + } + }, + { + "type": "table", + "title": "Branch summary", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (vcs_ref_head_name) (increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\", libtmux_spike=~\"$spike\"}[$__range]))", + "legendFormat": "requests", + "range": false, + "instant": true, + "refId": "A", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (vcs_ref_head_name) (increase(tmux_inlined_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\", libtmux_spike=~\"$spike\"}[$__range]))", + "legendFormat": "inlined", + "range": false, + "instant": true, + "refId": "B", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, vcs_ref_head_name) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\", libtmux_spike=~\"$spike\"}[$__range])))", + "legendFormat": "p95", + "range": false, + "instant": true, + "refId": "C", + "format": "table" + } + ], + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "job": true, + "instance": true + } + } + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto" + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "footer": { + "show": false + } + }, + "id": 8, + "gridPos": { + "x": 0, + "y": 27, + "w": 24, + "h": 8 + } + } + ] +} diff --git a/scripts/lgtm/dashboards/libtmux-overview.json b/scripts/lgtm/dashboards/libtmux-overview.json index 5d936b27c..ea07895ae 100644 --- a/scripts/lgtm/dashboards/libtmux-overview.json +++ b/scripts/lgtm/dashboards/libtmux-overview.json @@ -40,6 +40,29 @@ "text": "All", "value": "$__all" } + }, + { + "name": "branch", + "label": "Branch", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(tmux_requests_total, vcs_ref_head_name)", + "refId": "var-branch" + }, + "refresh": 2, + "sort": 1, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } } ] }, @@ -88,7 +111,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range]))", + "expr": "sum(increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range]))", "legendFormat": "__auto", "range": false, "instant": true, @@ -140,7 +163,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(increase(tmux_commands_total{tmux_lane=~\"$lane\"}[$__range]))", + "expr": "sum(increase(tmux_commands_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range]))", "legendFormat": "__auto", "range": false, "instant": true, @@ -192,7 +215,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "100 * sum(increase(tmux_inlined_total{tmux_lane=~\"$lane\"}[$__range])) / clamp_min(sum(increase(tmux_commands_total{tmux_lane=~\"$lane\"}[$__range])), 1)", + "expr": "100 * sum(increase(tmux_inlined_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range])) / clamp_min(sum(increase(tmux_commands_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range])), 1)", "legendFormat": "__auto", "range": false, "instant": true, @@ -244,7 +267,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "100 * sum(increase(tmux_failures_total{tmux_lane=~\"$lane\"}[$__range])) / clamp_min(sum(increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range])), 1)", + "expr": "100 * sum(increase(tmux_failures_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range])) / clamp_min(sum(increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range])), 1)", "legendFormat": "__auto", "range": false, "instant": true, @@ -326,7 +349,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_lane) (rate(tmux_requests_total{tmux_lane=~\"$lane\"}[$__rate_interval]))", + "expr": "sum by (tmux_lane) (rate(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__rate_interval]))", "legendFormat": "{{tmux_lane}}", "range": true, "instant": false, @@ -384,7 +407,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le, tmux_lane) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le, tmux_lane) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__rate_interval])))", "legendFormat": "{{tmux_lane}}", "range": true, "instant": false, @@ -443,7 +466,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (le) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__rate_interval]))", + "expr": "sum by (le) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__rate_interval]))", "legendFormat": "{{le}}", "range": true, "instant": false, @@ -501,7 +524,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_lane) (rate(tmux_failures_total{tmux_lane=~\"$lane\"}[$__rate_interval]))", + "expr": "sum by (tmux_lane) (rate(tmux_failures_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__rate_interval]))", "legendFormat": "{{tmux_lane}}", "range": true, "instant": false, diff --git a/scripts/lgtm/dashboards/libtmux-transports.json b/scripts/lgtm/dashboards/libtmux-transports.json index c727364b2..50c6de033 100644 --- a/scripts/lgtm/dashboards/libtmux-transports.json +++ b/scripts/lgtm/dashboards/libtmux-transports.json @@ -40,6 +40,29 @@ "text": "All", "value": "$__all" } + }, + { + "name": "branch", + "label": "Branch", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(tmux_requests_total, vcs_ref_head_name)", + "refId": "var-branch" + }, + "refresh": 2, + "sort": 1, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } } ] }, @@ -88,7 +111,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_lane) (increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range]))", + "expr": "sum by (tmux_lane) (increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range]))", "legendFormat": "{{tmux_lane}}", "range": false, "instant": true, @@ -147,7 +170,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_lane) (increase(tmux_commands_total{tmux_lane=~\"$lane\"}[$__range])) / clamp_min(sum by (tmux_lane) (increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range])), 1)", + "expr": "sum by (tmux_lane) (increase(tmux_commands_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range])) / clamp_min(sum by (tmux_lane) (increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range])), 1)", "legendFormat": "{{tmux_lane}}", "range": true, "instant": false, @@ -205,7 +228,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_lane) (rate(tmux_inlined_total{tmux_lane=~\"$lane\"}[$__rate_interval]))", + "expr": "sum by (tmux_lane) (rate(tmux_inlined_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__rate_interval]))", "legendFormat": "{{tmux_lane}}", "range": true, "instant": false, @@ -276,7 +299,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by (le, tmux_lane) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.5, sum by (le, tmux_lane) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__rate_interval])))", "legendFormat": "{{tmux_lane}}", "range": true, "instant": false, @@ -334,7 +357,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le, tmux_lane) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le, tmux_lane) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__rate_interval])))", "legendFormat": "{{tmux_lane}}", "range": true, "instant": false, @@ -392,7 +415,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, tmux_lane) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le, tmux_lane) (rate(tmux_command_duration_seconds_bucket{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__rate_interval])))", "legendFormat": "{{tmux_lane}}", "range": true, "instant": false, @@ -464,7 +487,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_lane) (increase(tmux_requests_total{tmux_lane=~\"$lane\"}[$__range]))", + "expr": "sum by (tmux_lane) (increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range]))", "legendFormat": "requests", "range": false, "instant": true, @@ -477,7 +500,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_lane) (increase(tmux_commands_total{tmux_lane=~\"$lane\"}[$__range]))", + "expr": "sum by (tmux_lane) (increase(tmux_commands_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range]))", "legendFormat": "commands", "range": false, "instant": true, @@ -490,7 +513,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (tmux_lane) (increase(tmux_inlined_total{tmux_lane=~\"$lane\"}[$__range]))", + "expr": "sum by (tmux_lane) (increase(tmux_inlined_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range]))", "legendFormat": "inlined", "range": false, "instant": true, diff --git a/scripts/lgtm/generate_dashboards.py b/scripts/lgtm/generate_dashboards.py index 12160ab63..98bbd9398 100644 --- a/scripts/lgtm/generate_dashboards.py +++ b/scripts/lgtm/generate_dashboards.py @@ -27,9 +27,12 @@ PYROSCOPE: dict[str, str] = {"type": "grafana-pyroscope-datasource", "uid": "pyroscope"} SERVICE = "libtmux-engines" -# Every panel filters by the lane template variable, so one board serves both -# "all transports together" and "this transport alone". -LANE = 'tmux_lane=~"$lane"' +# Every panel's selector, so filtering is uniform and adding a dimension here +# reaches all of them at once. Branch is in the default scope because the usual +# question is "did this branch change anything", and leaving it out silently +# mixes two branches into one line. +SCOPE = 'tmux_lane=~"$lane", vcs_ref_head_name=~"$branch"' +LANE = SCOPE # queries below read naturally as "the current scope" TAGS = ["libtmux", "generated"] BUCKET = "tmux_command_duration_seconds_bucket" @@ -190,18 +193,18 @@ def add(self, panel: dict[str, t.Any], *, w: int = 12, h: int = 8) -> None: panel["gridPos"] = self._place(w, h) self._panels.append(panel) - def lane_variable(self) -> None: - """Add the transport selector every board filters on.""" + def variable(self, name: str, label: str, metric_label: str) -> None: + """Add a multi-select variable populated from a metric's label values.""" self._templates.append( { - "name": "lane", - "label": "Transport", + "name": name, + "label": label, "type": "query", "datasource": PROM, "query": { "qryType": 1, - "query": "label_values(tmux_requests_total, tmux_lane)", - "refId": "var-lane", + "query": f"label_values(tmux_requests_total, {metric_label})", + "refId": f"var-{name}", }, "refresh": 2, "sort": 1, @@ -212,6 +215,11 @@ def lane_variable(self) -> None: } ) + def scope_variables(self) -> None: + """Add the selectors every board shares: transport and branch.""" + self.variable("lane", "Transport", "tmux_lane") + self.variable("branch", "Branch", "vcs_ref_head_name") + def to_dict(self) -> dict[str, t.Any]: """Render the dashboard envelope.""" return { @@ -466,7 +474,7 @@ def build_overview() -> Board: "trace behind it." ), ) - board.lane_variable() + board.scope_variables() board.row("Totals") board.add( @@ -609,7 +617,7 @@ def build_transports() -> Board: "operations, different dispatch cost." ), ) - board.lane_variable() + board.scope_variables() board.row("Share of work") board.add( @@ -719,7 +727,7 @@ def build_commands() -> Board: "libtmux / Commands", description="Which tmux commands the workload issues, and what they cost.", ) - board.lane_variable() + board.scope_variables() board.row("Command mix") board.add( @@ -810,7 +818,160 @@ def build_commands() -> Board: return board -BUILDERS = (build_overview, build_transports, build_commands) +def build_compare() -> Board: + """Build the board for comparing runs, branches, and spikes. + + The other boards answer "what is happening". This one answers "is this + different from that", which is the question the identity attributes exist + to serve: same panels, grouped by whichever axis is being compared. + """ + board = Board( + "libtmux-compare", + "libtmux / Compare", + description=( + "One run against another, one branch against another. Pick a spike " + "to scope the comparison to a single experiment." + ), + time_from="now-6h", + ) + board.scope_variables() + board.variable("spike", "Spike", "libtmux_spike") + + scoped = f'{SCOPE}, libtmux_spike=~"$spike"' + + def by(label: str, metric: str) -> str: + return f"sum by ({label}) (increase({metric}{{{scoped}}}[$__range]))" + + def quantile(quantile_value: float, label: str) -> str: + return ( + f"histogram_quantile({quantile_value}, sum by (le, {label}) " + f"(rate({BUCKET}{{{scoped}}}[$__range])))" + ) + + board.row("By run") + board.add( + table( + "Runs in range", + [ + target( + by("libtmux_run_id", "tmux_requests_total"), + "requests", + fmt="table", + instant=True, + ref="A", + ), + target( + by("libtmux_run_id", "tmux_commands_total"), + "commands", + fmt="table", + instant=True, + ref="B", + ), + target( + by("libtmux_run_id", "tmux_failures_total"), + "failures", + fmt="table", + instant=True, + ref="C", + ), + target( + quantile(0.95, "libtmux_run_id"), + "p95", + fmt="table", + instant=True, + ref="D", + ), + ], + description="Every run in the window, side by side.", + ), + w=24, + h=9, + ) + board.add( + timeseries( + "p95 by run", + [target(quantile(0.95, "libtmux_run_id"), "{{libtmux_run_id}}")], + unit="s", + description="Did a run get slower than the one before it?", + ), + w=12, + h=8, + ) + board.add( + timeseries( + "Request rate by run", + [ + target( + f"sum by (libtmux_run_id) " + f"(rate(tmux_requests_total{{{scoped}}}[$__rate_interval]))", + "{{libtmux_run_id}}", + ) + ], + unit="reqps", + ), + w=12, + h=8, + ) + + board.row("By branch") + board.add( + timeseries( + "p95 by branch", + [target(quantile(0.95, "vcs_ref_head_name"), "{{vcs_ref_head_name}}")], + unit="s", + description="The regression check: one branch against another.", + ), + w=12, + h=8, + ) + board.add( + piechart( + "Requests by branch", + [ + target( + by("vcs_ref_head_name", "tmux_requests_total"), + "{{vcs_ref_head_name}}", + instant=True, + ) + ], + ), + w=12, + h=8, + ) + board.add( + table( + "Branch summary", + [ + target( + by("vcs_ref_head_name", "tmux_requests_total"), + "requests", + fmt="table", + instant=True, + ref="A", + ), + target( + by("vcs_ref_head_name", "tmux_inlined_total"), + "inlined", + fmt="table", + instant=True, + ref="B", + ), + target( + quantile(0.95, "vcs_ref_head_name"), + "p95", + fmt="table", + instant=True, + ref="C", + ), + ], + ), + w=24, + h=8, + ) + return board + + +BUILDERS = (build_overview, build_transports, build_commands, build_compare) def write_dashboards(out_dir: pathlib.Path) -> list[pathlib.Path]: diff --git a/scripts/lgtm/identity.py b/scripts/lgtm/identity.py new file mode 100644 index 000000000..7bfd82e78 --- /dev/null +++ b/scripts/lgtm/identity.py @@ -0,0 +1,245 @@ +"""Resolve who produced a telemetry run, and decide where each fact belongs. + +The hard part of adding branch, commit, worktree, spike, and test identity to +telemetry is not collecting it. It is deciding which signal carries which fact, +because the three signals fail in different ways when you get it wrong. + +Metrics + Every distinct attribute combination is a stored time series forever. A + commit SHA on a metric means a fresh set of series on every commit, growing + without bound, and the cost lands on whoever runs Prometheus next month. + Only dimensions you *compare across* belong here. +Traces + Attributes are per span and stored with it. Tempo is built for high + cardinality, so this is where the drill-down detail goes -- SHA, worktree + path, test case -- and TraceQL can filter on any of it. +Profiles + Pyroscope labels are per process, and a run is one process, so the full + static identity is free here. + +The dividing question for metrics is "would I ever group by this?", not "is it +interesting?". A SHA is never a comparison axis: each run has exactly one, so +grouping by SHA is grouping by run, which :data:`METRIC_KEYS` already allows +through the run id. Carrying it as well buys no query power and costs unbounded +series. That is the one place this deliberately diverges from what agentgrep's +otel-bootstrap branch does. + +Names follow OpenTelemetry semantic conventions where they exist -- the ``vcs.*`` +group for repository and ref, the ``test.*`` group for test identity -- so +anything already written against those conventions can read this without a +translation table. +""" + +from __future__ import annotations + +import os +import pathlib +import subprocess +import typing as t + +# Resource attributes copied onto every metric point, and the Prometheus label +# each becomes. Keep this list short and boring: these are comparison axes. +# +# Absent on purpose: vcs.ref.head.revision (one per run, so it adds no grouping +# power over the run id while multiplying series on every commit), the +# repository URL (constant, and it can carry a private host name), the worktree +# path (a local absolute path), and test identity (one series per test case). +METRIC_KEYS: tuple[tuple[str, str], ...] = ( + ("vcs.ref.head.name", "vcs_ref_head_name"), + ("libtmux.run_id", "libtmux_run_id"), + ("libtmux.spike", "libtmux_spike"), +) + +# Baggage keys copied onto spans by the processor in telemetry.py. These change +# within a process -- per test, per phase -- so they cannot be resource +# attributes, and they are far too high-cardinality for metrics. +BAGGAGE_KEYS: tuple[str, ...] = ( + "test.case.name", + "test.suite.name", + "libtmux.phase", +) + +_GIT_TIMEOUT = 5.0 + + +def _git(root: pathlib.Path, *args: str) -> list[str] | None: + """Run one git command, returning its output lines or ``None``.""" + try: + completed = subprocess.run( + ("git", "-C", str(root), *args), + capture_output=True, + text=True, + timeout=_GIT_TIMEOUT, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + if completed.returncode != 0: + return None + return completed.stdout.strip().splitlines() + + +def vcs_attributes(root: pathlib.Path | None = None) -> dict[str, str]: + """Resolve repository, ref, and worktree identity for *root*. + + One ``git rev-parse`` answers all of it in a couple of milliseconds, which + matters because this runs at process start on a developer's machine. When + the environment already names the ref -- CI usually does -- the git call is + skipped entirely. + + Returns + ------- + dict + OpenTelemetry ``vcs.*`` attributes plus ``libtmux.worktree``. Empty when + *root* is not a git repository. + """ + root = root or pathlib.Path(__file__).resolve().parents[2] + attributes: dict[str, str] = {} + + lines = _git( + root, + "rev-parse", + "--show-toplevel", + "--git-common-dir", + "HEAD", + "--abbrev-ref", + "HEAD", + ) + if not lines or len(lines) < 4: + return attributes + toplevel, common_dir, revision, ref = lines[0], lines[1], lines[2], lines[3] + + attributes["vcs.ref.head.revision"] = revision + if ref == "HEAD": + # Detached: prefer a tag, and fall back to the short revision so the + # dimension is never the literal string "HEAD" for every detached run. + described = _git(root, "describe", "--tags", "--exact-match", "HEAD") + if described: + attributes["vcs.ref.head.name"] = described[0] + attributes["vcs.ref.head.type"] = "tag" + else: + attributes["vcs.ref.head.name"] = revision[:12] + attributes["vcs.ref.head.type"] = "revision" + else: + attributes["vcs.ref.head.name"] = ref + attributes["vcs.ref.head.type"] = "branch" + + # The repository name comes from the common git dir, which every linked + # worktree shares, so worktrees of one repo group together rather than + # looking like separate projects. + attributes["vcs.repository.name"] = pathlib.Path(common_dir).resolve().parent.name + + # git-dir differs from git-common-dir only inside a linked worktree. A + # plain clone whose directory name differs from the repository name gets + # the label too, since that is how sibling checkouts are told apart. + git_dir = _git(root, "rev-parse", "--git-dir") + checkout = pathlib.Path(toplevel).name + linked = bool(git_dir) and ( + pathlib.Path(git_dir[0]).resolve() != pathlib.Path(common_dir).resolve() + ) + if linked or checkout != attributes["vcs.repository.name"]: + attributes["libtmux.worktree"] = checkout + + return attributes + + +def resolve( + *, + run_id: str, + spike: str | None = None, + service_name: str = "libtmux-engines", + root: pathlib.Path | None = None, + env: t.Mapping[str, str] | None = None, +) -> dict[str, str]: + """Build the full resource attribute set for one telemetry run. + + Parameters + ---------- + run_id : str + Identifies this run; the finest comparison axis metrics carry. + spike : str or None + Names an experiment, so several runs can be grouped and compared. + Falls back to ``LIBTMUX_SPIKE``. + service_name : str + OpenTelemetry service name. + root : pathlib.Path or None + Repository to inspect; defaults to this checkout. + env : Mapping or None + Environment to read overrides from; defaults to :data:`os.environ`. + + Returns + ------- + dict + Resource attributes, ready for ``Resource.create``. + """ + environ = os.environ if env is None else env + attributes: dict[str, str] = { + "service.name": service_name, + "libtmux.run_id": run_id, + } + attributes.update(vcs_attributes(root)) + + # An explicit ref wins over the checkout's own: CI checks out a detached + # HEAD but knows the branch the work belongs to. + for variable, key in ( + ("LIBTMUX_VCS_REF", "vcs.ref.head.name"), + ("LIBTMUX_VCS_REVISION", "vcs.ref.head.revision"), + ("LIBTMUX_WORKTREE", "libtmux.worktree"), + ): + value = environ.get(variable) + if value: + attributes[key] = value + + resolved_spike = spike or environ.get("LIBTMUX_SPIKE") + if resolved_spike: + attributes["libtmux.spike"] = resolved_spike + return attributes + + +def metric_attributes(resource: t.Mapping[str, str]) -> dict[str, str]: + """Select the bounded subset of *resource* that metrics may carry. + + Everything not in :data:`METRIC_KEYS` is dropped rather than renamed, so a + new resource attribute cannot silently become a new Prometheus label. That + is the whole safety property: growing the metric surface takes an edit + here, where the cardinality cost is written down. + + Examples + -------- + >>> metric_attributes({"vcs.ref.head.name": "main", "vcs.ref.head.revision": "abc"}) + {'vcs_ref_head_name': 'main'} + """ + selected: dict[str, str] = {} + for resource_key, label in METRIC_KEYS: + value = resource.get(resource_key) + if value: + selected[label] = value + return selected + + +def profile_tags(resource: t.Mapping[str, str]) -> dict[str, str]: + """Select Pyroscope tags: the static identity, minus anything path-like. + + A profile is one process, so the full identity costs nothing here. Absolute + paths are still excluded because they carry a home directory into a stored + label. + + Examples + -------- + >>> tags = profile_tags({ + ... "vcs.ref.head.name": "main", + ... "vcs.ref.head.revision": "abc123", + ... "libtmux.run_id": "r1", + ... }) + >>> sorted(tags) + ['libtmux_run_id', 'vcs_ref_head_name', 'vcs_ref_head_revision'] + """ + wanted = ( + "vcs.ref.head.name", + "vcs.ref.head.revision", + "vcs.repository.name", + "libtmux.worktree", + "libtmux.run_id", + "libtmux.spike", + ) + return {key.replace(".", "_"): resource[key] for key in wanted if resource.get(key)} diff --git a/scripts/lgtm/telemetry.py b/scripts/lgtm/telemetry.py index 3551d353d..99bf97191 100644 --- a/scripts/lgtm/telemetry.py +++ b/scripts/lgtm/telemetry.py @@ -14,10 +14,12 @@ from __future__ import annotations +import contextlib import time import typing as t -from opentelemetry import metrics, trace +import identity +from opentelemetry import baggage, context, metrics, trace from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter @@ -26,7 +28,7 @@ from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace import SpanProcessor, TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from libtmux.experimental.engines.control_mode import command_count @@ -56,6 +58,64 @@ ) +class BaggageSpanProcessor(SpanProcessor): + """Copy selected baggage entries onto every span as it starts. + + Baggage is how a value set once -- the test now running, the phase a + workload is in -- reaches spans created deep inside the engines, without + threading a parameter through call signatures that have no business knowing + about telemetry. + + Only the keys in :data:`identity.BAGGAGE_KEYS` are copied. Baggage + propagates to other processes, so mirroring all of it onto spans would let + an unrelated caller's entries silently become attributes here. + + The cost is one context read per span and nothing per tmux command, and + when no baggage is set the loop body never runs. + """ + + def on_start(self, span: t.Any, parent_context: t.Any = None) -> None: + """Stamp the active baggage onto a starting span.""" + entries = baggage.get_all(parent_context) + if not entries: + return + for key in identity.BAGGAGE_KEYS: + value = entries.get(key) + if value is not None: + span.set_attribute(key, str(value)) + + def on_end(self, span: t.Any) -> None: + """Nothing to do; export is another processor's job.""" + + def shutdown(self) -> None: + """Nothing to release.""" + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Nothing buffered.""" + del timeout_millis + return True + + +@contextlib.contextmanager +def scope(**entries: str) -> t.Iterator[None]: + """Attach *entries* to every span started inside the block. + + Examples + -------- + >>> with scope(**{"libtmux.phase": "warmup"}): + ... pass + """ + token = None + current = context.get_current() + for key, value in entries.items(): + current = baggage.set_baggage(key, value, context=current) + token = context.attach(current) + try: + yield + finally: + context.detach(token) + + class OTelSink: """Emit one span and one set of metric points per tmux command. @@ -71,15 +131,26 @@ class OTelSink: "_commands", "_duration", "_failures", + "_identity", "_inlined", "_lane", "_requests", "_tracer", ) - def __init__(self, tracer: t.Any, meter: t.Any, lane: str) -> None: + def __init__( + self, + tracer: t.Any, + meter: t.Any, + lane: str, + identity_labels: t.Mapping[str, str] | None = None, + ) -> None: self._tracer = tracer self._lane = lane + # Resolved once per process, merged into every point. Building the dict + # per command would allocate on the hot path for a value that cannot + # change. + self._identity = dict(identity_labels or {}) self._requests = meter.create_counter( "tmux.requests", description="Requests dispatched to an engine." ) @@ -101,7 +172,7 @@ def __init__(self, tracer: t.Any, meter: t.Any, lane: str) -> None: ) def _attrs(self, command: str) -> dict[str, str]: - return {"tmux.lane": self._lane, "tmux.command": command} + return {"tmux.lane": self._lane, "tmux.command": command, **self._identity} def before_command(self, request: CommandRequest) -> tuple[t.Any, float, str]: """Open a span and count the request, returning per-command state.""" @@ -168,6 +239,12 @@ class Telemetry(t.NamedTuple): Meter the sinks build instruments from. handler : logging.Handler Handler that ships records to Loki with trace context attached. + resource_attributes : dict + Full identity for this run; on every span and log by construction. + metric_labels : dict + The bounded subset metrics may carry, already renamed for Prometheus. + profile_tags : dict + Static identity for Pyroscope. """ tracer_provider: t.Any @@ -176,6 +253,9 @@ class Telemetry(t.NamedTuple): tracer: t.Any meter: t.Any handler: t.Any + resource_attributes: dict[str, str] + metric_labels: dict[str, str] + profile_tags: dict[str, str] def shutdown(self) -> None: """Flush and stop every provider, in export order.""" @@ -187,7 +267,13 @@ def shutdown(self) -> None: self.logger_provider.shutdown() -def build(endpoint: str, *, run_id: str, export_interval_ms: int = 2000) -> Telemetry: +def build( + endpoint: str, + *, + run_id: str, + spike: str | None = None, + export_interval_ms: int = 2000, +) -> Telemetry: """Wire OTLP exporters for traces, metrics, and logs. Parameters @@ -196,6 +282,8 @@ def build(endpoint: str, *, run_id: str, export_interval_ms: int = 2000) -> Tele OTLP HTTP base URL, for example ``http://127.0.0.1:4318``. run_id : str Identifies one smoke run, so a dashboard can isolate it. + spike : str or None + Names an experiment several runs belong to, for grouped comparison. export_interval_ms : int How often metrics are pushed. Short, because a smoke run is short. @@ -204,9 +292,12 @@ def build(endpoint: str, *, run_id: str, export_interval_ms: int = 2000) -> Tele Telemetry Providers and handles; call :meth:`Telemetry.shutdown` when done. """ - resource = Resource.create({"service.name": SERVICE_NAME, "libtmux.run_id": run_id}) + attributes = identity.resolve(run_id=run_id, spike=spike, service_name=SERVICE_NAME) + resource = Resource.create(dict(attributes)) tracer_provider = TracerProvider(resource=resource) + # Baggage first: it must stamp a span before the batch processor sees it. + tracer_provider.add_span_processor(BaggageSpanProcessor()) tracer_provider.add_span_processor( BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")) ) @@ -235,4 +326,7 @@ def build(endpoint: str, *, run_id: str, export_interval_ms: int = 2000) -> Tele tracer=trace.get_tracer("libtmux.engines"), meter=metrics.get_meter("libtmux.engines"), handler=LoggingHandler(logger_provider=logger_provider), + resource_attributes=attributes, + metric_labels=identity.metric_attributes(attributes), + profile_tags=identity.profile_tags(attributes), ) diff --git a/scripts/otel_acceptance.py b/scripts/otel_acceptance.py index b03fc1eb6..6a1355d0f 100644 --- a/scripts/otel_acceptance.py +++ b/scripts/otel_acceptance.py @@ -90,7 +90,12 @@ def expand(expr: str) -> str: expr.replace("$__rate_interval", RATE_INTERVAL) .replace("$__interval", RATE_INTERVAL) .replace("$__range", DASHBOARD_RANGE) + # Every scope variable defaults to All, which Grafana interpolates to + # the allValue regex. .replace("$lane", ".*") + .replace("$branch", ".*") + .replace("$spike", ".*") + .replace("$run", ".*") ) diff --git a/scripts/otel_smoke.py b/scripts/otel_smoke.py index aaa1d2a43..5c89475a1 100644 --- a/scripts/otel_smoke.py +++ b/scripts/otel_smoke.py @@ -154,6 +154,11 @@ def main(argv: list[str] | None = None) -> int: """Run every lane under full telemetry and print the local counts.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--run-id", default=f"smoke-{uuid.uuid4().hex[:8]}") + parser.add_argument( + "--spike", + default=None, + help="name an experiment so several runs group together (or LIBTMUX_SPIKE)", + ) parser.add_argument( "--seconds", type=float, default=4.0, help="workload duration per lane" ) @@ -176,7 +181,7 @@ def main(argv: list[str] | None = None) -> int: os.environ.pop("TMUX", None) os.environ.pop("TMUX_PANE", None) - signals = telemetry.build(args.otlp, run_id=args.run_id) + signals = telemetry.build(args.otlp, run_id=args.run_id, spike=args.spike) logging.basicConfig(level=logging.INFO, handlers=[signals.handler], force=True) try: @@ -187,7 +192,10 @@ def main(argv: list[str] | None = None) -> int: server_address=args.pyroscope, sample_rate=100, upload_interval=3, - tags={"run_id": args.run_id}, + # The full static identity: a profile is one process, so branch, + # revision, worktree, and spike cost nothing extra here and make + # two runs directly comparable in Pyroscope. + tags=signals.profile_tags, ) profiling = True except Exception as error: # noqa: BLE001 - profiling is optional @@ -211,9 +219,12 @@ def main(argv: list[str] | None = None) -> int: engine = instrument( factory(), counts, - telemetry.OTelSink(signals.tracer, signals.meter, lane), + telemetry.OTelSink( + signals.tracer, signals.meter, lane, signals.metric_labels + ), ) - run_sync(engine, args.seconds) + with telemetry.scope(**{"libtmux.phase": f"{lane}-sync"}): + run_sync(engine, args.seconds) totals[lane] = lane_totals(counts) logger.info("lane finished", extra={"lane": lane, **totals[lane]}) @@ -226,7 +237,9 @@ def main(argv: list[str] | None = None) -> int: engine = instrument( factory(), counts, - telemetry.OTelSink(signals.tracer, signals.meter, lane), + telemetry.OTelSink( + signals.tracer, signals.meter, lane, signals.metric_labels + ), ) async def drive(engine: t.Any = engine) -> None: @@ -237,7 +250,8 @@ async def drive(engine: t.Any = engine) -> None: if hasattr(inner, "aclose"): await inner.aclose() - asyncio.run(drive()) + with telemetry.scope(**{"libtmux.phase": f"{lane}-async"}): + asyncio.run(drive()) totals[lane] = lane_totals(counts) logger.info("lane finished", extra={"lane": lane, **totals[lane]}) finally: @@ -266,7 +280,9 @@ async def drive(engine: t.Any = engine) -> None: f" {lane:<18}{row['requests']:>10}{row['tmux_commands']:>10}" f"{row['inlined']:>10}{row['elapsed_ms']:>11}" ) - print(f"\n run_id={args.run_id} service={telemetry.SERVICE_NAME}") + ref = signals.resource_attributes.get("vcs.ref.head.name", "?") + worktree = signals.resource_attributes.get("libtmux.worktree", "-") + print(f"\n run_id={args.run_id} branch={ref} worktree={worktree}") print(f" exported to {args.otlp} and {args.pyroscope}") return 0 diff --git a/tests/test_lgtm_dashboards.py b/tests/test_lgtm_dashboards.py index 7912f714d..b4bb817fc 100644 --- a/tests/test_lgtm_dashboards.py +++ b/tests/test_lgtm_dashboards.py @@ -177,3 +177,57 @@ def test_no_panel_reads_a_counter_without_a_window() -> None: f"range window and will blank out once the series goes " f"stale: {expr}" ) + + +def test_metric_labels_stay_a_closed_low_cardinality_set() -> None: + """Metrics may carry only the dimensions worth grouping by. + + Every distinct label combination is a Prometheus series kept forever, so + this set is a budget, not a convenience. A commit SHA is the tempting + mistake: each run has exactly one, so grouping by SHA is grouping by run, + which the run id already allows -- it would buy no query power while + adding a fresh set of series on every commit. + + Detail like the SHA, the worktree path, and test identity is not lost; it + rides on spans and profiles, where high cardinality is expected and the + drill-down actually happens. + """ + spec = importlib.util.spec_from_file_location("identity", _LGTM / "identity.py") + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + assert {label for _key, label in module.METRIC_KEYS} == { + "vcs_ref_head_name", + "libtmux_run_id", + "libtmux_spike", + } + + resource = { + "vcs.ref.head.name": "main", + "vcs.ref.head.revision": "deadbeef", + "vcs.repository.name": "libtmux", + "libtmux.worktree": "/home/someone/checkout", + "libtmux.run_id": "r1", + } + labels = module.metric_attributes(resource) + assert "vcs_ref_head_revision" not in labels + assert "libtmux_worktree" not in labels + assert labels["vcs_ref_head_name"] == "main" + + # The same facts must still reach profiles, where they are affordable. + tags = module.profile_tags(resource) + assert "vcs_ref_head_revision" in tags + + +def test_every_scope_variable_used_is_defined() -> None: + """A panel filtering on an undefined variable silently returns nothing.""" + for board in _boards(): + names = {var["name"] for var in board["templating"]["list"]} + serialized = json.dumps(board["panels"]) + for variable in ("lane", "branch", "spike", "run"): + if f"${variable}" in serialized: + assert variable in names, ( + f"{board['uid']} filters on ${variable} without defining it" + ) From d2fb9e384590cb2220d147b95a6ecca73cf92e09 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 09:46:34 -0500 Subject: [PATCH 04/24] Bench(feat[lgtm]): Shape load with rampa and follow Grafana's dashboard guidance why: A fixed-worker loop cannot find saturation. As latency rises the workers slow with it, offered load falls, and the graph bends instead of breaking, so the steady workload could say how fast a transport is but never where it stops keeping up. Separately, five boards with no entry point is how dashboard sprawl starts: without directed browsing, finding the right one is guesswork and the fix people reach for is duplicating it. what: - Add scripts/lgtm/load_tmux.py and just otel-load, driving the engines under rampa's ramping-arrival-rate; the same control-mode engine measures p99 around 2 ms steady and around 16 ms at the top of the ramp - Keep telemetry on this project's sink rather than a rampa output backend: rampa's own backend would export under its service name and metric vocabulary, so a load-shaped run would arrive as a second account of the same work rather than a comparable one - Add a Home board naming what each board answers, and link Overview's panels down to the board that explains them - Reorganize Overview as rate, errors, and duration, since an engine is a service and RED is the frame its caller thinks in - Drop stacking and relax refresh to a minute: the data arrives in bursts from short runs, so a thirty-second poll buys nothing - Teach the dashboard tests that a text panel is documentation and is not expected to query anything --- CHANGES | 11 + justfile | 5 + pyproject.toml | 6 + scripts/lgtm/README.md | 36 +- scripts/lgtm/dashboards/libtmux-commands.json | 10 +- scripts/lgtm/dashboards/libtmux-compare.json | 2 +- scripts/lgtm/dashboards/libtmux-home.json | 311 ++++++++ scripts/lgtm/dashboards/libtmux-overview.json | 114 ++- .../lgtm/dashboards/libtmux-transports.json | 2 +- scripts/lgtm/generate_dashboards.py | 162 ++++- scripts/lgtm/load_tmux.py | 183 +++++ tests/test_lgtm_dashboards.py | 18 +- uv.lock | 673 ++++++++++++++++++ 13 files changed, 1474 insertions(+), 59 deletions(-) create mode 100644 scripts/lgtm/dashboards/libtmux-home.json create mode 100644 scripts/lgtm/load_tmux.py diff --git a/CHANGES b/CHANGES index 1713844bf..adfcfb55c 100644 --- a/CHANGES +++ b/CHANGES @@ -298,6 +298,17 @@ Values that change within a process, such as the phase a workload is in, travel as OpenTelemetry baggage and are copied onto spans by a processor instead of being threaded through engine calls. +`just otel-load` drives the engines under a ramping arrival rate, an open model +that keeps offering work regardless of latency and so exposes where a transport +saturates -- something a fixed-worker loop hides, because it slows down along +with the system. The schedule comes from rampa; the telemetry still comes from +this project's own sink, so a load-shaped run is directly comparable with every +other run rather than speaking a second vocabulary. + +The dashboards follow Grafana's guidance: a Home board says which board answers +which question, Overview is organized as rate, errors, and duration, and its +panels link down to the board that explains them. + #### CI actions updated to current majors Workflow actions moved to their current major releases: `actions/checkout` v7, diff --git a/justfile b/justfile index 57e4f5efc..b77ccfecb 100644 --- a/justfile +++ b/justfile @@ -174,3 +174,8 @@ otel-acceptance *args: [group: 'otel'] otel-verify: uv run --group otel python scripts/otel_acceptance.py --start-stack --smoke + +# Drive the engines under a load shape (ramping arrival rate) via rampa +[group: 'otel'] +otel-load *args: + uv run --group otel --group load rampa run scripts/lgtm/load_tmux.py {{ args }} diff --git a/pyproject.toml b/pyproject.toml index e7562afcd..16b4ee6b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,12 @@ otel = [ "opentelemetry-exporter-otlp-proto-http", "pyroscope-io", ] +# Load shaping for scripts/lgtm/load_tmux.py (just otel-load). Separate from +# `otel` because rampa needs a newer Python than libtmux supports, so the +# marker keeps `uv sync` working on every version libtmux targets. +load = [ + "rampa; python_version >= '3.14'", +] dev = [ # Docs (via gp-sphinx) "gp-sphinx==0.1.0a37", diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md index 88646d170..c8d62c667 100644 --- a/scripts/lgtm/README.md +++ b/scripts/lgtm/README.md @@ -146,10 +146,12 @@ set the loop body never runs. ## Dashboards -Four boards, provisioned into the `libtmux` folder: +Five boards, provisioned into the `libtmux` folder: -`libtmux / Overview` is throughput, latency, failures, and the trace, log, and -profile panels side by side. `libtmux / Transports` compares the four transports +`libtmux / Home` is the entry point: which board answers which question, with +links. `libtmux / Overview` follows the RED method -- rate, errors, duration -- +because an engine is a service, and its panels link down to the board that +explains them. `libtmux / Transports` compares the four transports against each other. `libtmux / Commands` breaks the same work down by tmux command. `libtmux / Compare` answers "is this different from that", grouping the same measurements by run and by branch. @@ -171,6 +173,34 @@ the committed JSON differs from what the generator produces, so the two cannot diverge quietly. Editing a board in the Grafana UI is fine for exploring; move the change into the generator to keep it. +## Load shaping: why rampa and not k6 + +`just otel-smoke` runs flat out for a fixed duration with a fixed number of +workers. That is a closed loop, and a closed loop hides saturation: as latency +rises the workers slow down with it, offered load falls, and the graph bends +politely instead of breaking. + +`just otel-load` uses [rampa](https://github.com/tony/rampa) for the shapes +that expose it, notably `ramping-arrival-rate` -- an open model that keeps +issuing commands at a target rate whether or not the previous ones finished, so +latency climbs on its own once a transport saturates. The difference is +visible: the same control-mode engine measures p99 around 2 ms under the steady +workload and around 16 ms at the top of the ramp. + +k6 was the obvious alternative and is the wrong tool here. Its value is HTTP +load at scale, and there is no HTTP surface in front of the engines -- driving +Python from k6 means a subprocess or an HTTP shim per iteration, which measures +the shim. rampa is Python, so it drives the engines in-process, and it +implements the same six executor models k6 defines. + +rampa provides the schedule; telemetry still comes from this project's own +sink. That split is deliberate rather than a workaround: rampa's own OTLP +backend would export under its service name and its metric vocabulary, so a +load-shaped run would arrive as a second, parallel account of the same work. +Going through the sink instead means it lands under the same metric names and +the same branch, worktree, and spike labels as every other run, and the two are +directly comparable. + ## Why the acceptance check exists A dashboard that renders is not a dashboard that works. A panel whose query diff --git a/scripts/lgtm/dashboards/libtmux-commands.json b/scripts/lgtm/dashboards/libtmux-commands.json index dff961a8b..8594239ff 100644 --- a/scripts/lgtm/dashboards/libtmux-commands.json +++ b/scripts/lgtm/dashboards/libtmux-commands.json @@ -11,7 +11,7 @@ "version": 1, "editable": true, "graphTooltip": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-1h", "to": "now" @@ -184,13 +184,9 @@ }, "unit": "reqps", "custom": { - "fillOpacity": 18, + "fillOpacity": 10, "showPoints": "never", - "lineWidth": 2, - "stacking": { - "mode": "normal", - "group": "A" - } + "lineWidth": 2 } }, "overrides": [] diff --git a/scripts/lgtm/dashboards/libtmux-compare.json b/scripts/lgtm/dashboards/libtmux-compare.json index eb5746d47..29c5adb0f 100644 --- a/scripts/lgtm/dashboards/libtmux-compare.json +++ b/scripts/lgtm/dashboards/libtmux-compare.json @@ -11,7 +11,7 @@ "version": 1, "editable": true, "graphTooltip": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-6h", "to": "now" diff --git a/scripts/lgtm/dashboards/libtmux-home.json b/scripts/lgtm/dashboards/libtmux-home.json new file mode 100644 index 000000000..265212173 --- /dev/null +++ b/scripts/lgtm/dashboards/libtmux-home.json @@ -0,0 +1,311 @@ +{ + "uid": "libtmux-home", + "title": "libtmux / Home", + "description": "Start here. Which board answers which question.", + "tags": [ + "libtmux", + "generated" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "graphTooltip": 1, + "refresh": "", + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "lane", + "label": "Transport", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(tmux_requests_total, tmux_lane)", + "refId": "var-lane" + }, + "refresh": 2, + "sort": 1, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + }, + { + "name": "branch", + "label": "Branch", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(tmux_requests_total, vcs_ref_head_name)", + "refId": "var-branch" + }, + "refresh": 2, + "sort": 1, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "links": [ + { + "title": "libtmux dashboards", + "type": "dashboards", + "tags": [ + "libtmux" + ], + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "icon": "external link" + } + ], + "annotations": { + "list": [] + }, + "panels": [ + { + "type": "text", + "title": "", + "datasource": null, + "options": { + "mode": "markdown", + "content": "# libtmux engine observability\n\nTelemetry comes from the engine instrumentation seam, so the exporters are ordinary sinks and the engines are untouched.\n\n| Board | Answers |\n| --- | --- |\n| [Overview](/d/libtmux-overview) | Is work flowing, and is it healthy? Rate, errors, duration, plus traces, logs and profiles. |\n| [Transports](/d/libtmux-transports) | Which transport is responsible? Subprocess against control mode, sync against async. |\n| [Commands](/d/libtmux-commands) | Which tmux command is responsible? |\n| [Compare](/d/libtmux-compare) | Did this run or branch change anything? |\n\n**Producing data**: `just otel-smoke` for a steady workload, `just otel-load` for a ramping arrival rate that exposes saturation. `just otel-acceptance` checks every panel still has data.\n\nEvery board is generated by `scripts/lgtm/generate_dashboards.py`; edits made in this UI are overwritten on the next `just otel-up`." + }, + "transparent": true, + "id": 1, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 13 + } + }, + { + "type": "row", + "title": "Health right now", + "collapsed": false, + "id": 2, + "gridPos": { + "x": 0, + "y": 13, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "stat", + "title": "Requests in range", + "description": "Across every transport in the selected window.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range]))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "id": 3, + "gridPos": { + "x": 0, + "y": 14, + "w": 8, + "h": 5 + }, + "links": [ + { + "title": "Open Overview", + "url": "/d/libtmux-overview?${__url_time_range}&var-lane=${lane:queryparam}&var-branch=${branch:queryparam}", + "targetBlank": false + } + ] + }, + { + "type": "stat", + "title": "Failure share", + "description": "Requests tmux rejected.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "100 * sum(increase(tmux_failures_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range])) / clamp_min(sum(increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range])), 1)", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "id": 4, + "gridPos": { + "x": 8, + "y": 14, + "w": 8, + "h": 5 + }, + "links": [ + { + "title": "Open Commands", + "url": "/d/libtmux-commands?${__url_time_range}&var-lane=${lane:queryparam}&var-branch=${branch:queryparam}", + "targetBlank": false + } + ] + }, + { + "type": "stat", + "title": "Branches reporting", + "description": "How many branches have data in this window.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(count by (vcs_ref_head_name) (sum by (vcs_ref_head_name) (increase(tmux_requests_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"}[$__range]))))", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "id": 5, + "gridPos": { + "x": 16, + "y": 14, + "w": 8, + "h": 5 + }, + "links": [ + { + "title": "Open Compare", + "url": "/d/libtmux-compare?${__url_time_range}&var-lane=${lane:queryparam}&var-branch=${branch:queryparam}", + "targetBlank": false + } + ] + } + ] +} diff --git a/scripts/lgtm/dashboards/libtmux-overview.json b/scripts/lgtm/dashboards/libtmux-overview.json index ea07895ae..006b50374 100644 --- a/scripts/lgtm/dashboards/libtmux-overview.json +++ b/scripts/lgtm/dashboards/libtmux-overview.json @@ -11,7 +11,7 @@ "version": 1, "editable": true, "graphTooltip": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-1h", "to": "now" @@ -84,14 +84,31 @@ }, "panels": [ { - "type": "row", - "title": "Totals", - "collapsed": false, + "type": "text", + "title": "", + "datasource": null, + "options": { + "mode": "markdown", + "content": "**Rate, Errors, Duration** for the tmux engines. RED is the right frame here because an engine is a service: the caller cares how much work goes through, how much of it fails, and how long it takes.\n\nPanels link to the board that explains them. Use **Transports** for which transport is responsible, **Commands** for which tmux command, and **Compare** for whether a branch or run changed anything." + }, + "transparent": true, "id": 1, "gridPos": { "x": 0, "y": 0, "w": 24, + "h": 4 + } + }, + { + "type": "row", + "title": "Rate", + "collapsed": false, + "id": 2, + "gridPos": { + "x": 0, + "y": 4, + "w": 24, "h": 1 }, "panels": [] @@ -140,10 +157,10 @@ "values": false } }, - "id": 2, + "id": 3, "gridPos": { "x": 0, - "y": 1, + "y": 5, "w": 6, "h": 5 } @@ -192,14 +209,27 @@ "values": false } }, - "id": 3, + "id": 4, "gridPos": { "x": 6, - "y": 1, + "y": 5, "w": 6, "h": 5 } }, + { + "type": "row", + "title": "Errors", + "collapsed": false, + "id": 5, + "gridPos": { + "x": 0, + "y": 10, + "w": 24, + "h": 1 + }, + "panels": [] + }, { "type": "stat", "title": "Inlined share", @@ -244,10 +274,10 @@ "values": false } }, - "id": 4, + "id": 6, "gridPos": { - "x": 12, - "y": 1, + "x": 0, + "y": 11, "w": 6, "h": 5 } @@ -313,22 +343,22 @@ "values": false } }, - "id": 5, + "id": 7, "gridPos": { - "x": 18, - "y": 1, + "x": 6, + "y": 11, "w": 6, "h": 5 } }, { "type": "row", - "title": "Throughput and latency", + "title": "Rate over time", "collapsed": false, - "id": 6, + "id": 8, "gridPos": { "x": 0, - "y": 6, + "y": 16, "w": 24, "h": 1 }, @@ -384,13 +414,20 @@ "sort": "desc" } }, - "id": 7, + "id": 9, "gridPos": { "x": 0, - "y": 7, + "y": 17, "w": 12, "h": 8 - } + }, + "links": [ + { + "title": "Break down by transport", + "url": "/d/libtmux-transports?${__url_time_range}&var-lane=${lane:queryparam}&var-branch=${branch:queryparam}", + "targetBlank": false + } + ] }, { "type": "timeseries", @@ -443,10 +480,10 @@ "sort": "desc" } }, - "id": 8, + "id": 10, "gridPos": { "x": 12, - "y": 7, + "y": 17, "w": 12, "h": 8 } @@ -501,10 +538,10 @@ }, "overrides": [] }, - "id": 9, + "id": 11, "gridPos": { "x": 0, - "y": 15, + "y": 25, "w": 12, "h": 8 } @@ -559,22 +596,29 @@ "sort": "desc" } }, - "id": 10, + "id": 12, "gridPos": { "x": 12, - "y": 15, + "y": 25, "w": 12, "h": 8 - } + }, + "links": [ + { + "title": "Break down by command", + "url": "/d/libtmux-commands?${__url_time_range}&var-lane=${lane:queryparam}&var-branch=${branch:queryparam}", + "targetBlank": false + } + ] }, { "type": "row", "title": "Traces, logs, and profiles", "collapsed": false, - "id": 11, + "id": 13, "gridPos": { "x": 0, - "y": 23, + "y": 33, "w": 24, "h": 1 }, @@ -604,10 +648,10 @@ "options": { "showHeader": true }, - "id": 12, + "id": 14, "gridPos": { "x": 0, - "y": 24, + "y": 34, "w": 12, "h": 8 } @@ -640,10 +684,10 @@ "wrapLogMessage": true, "enableLogDetails": true }, - "id": 13, + "id": 15, "gridPos": { "x": 12, - "y": 24, + "y": 34, "w": 12, "h": 8 } @@ -670,10 +714,10 @@ } ], "options": {}, - "id": 14, + "id": 16, "gridPos": { "x": 0, - "y": 32, + "y": 42, "w": 24, "h": 11 } diff --git a/scripts/lgtm/dashboards/libtmux-transports.json b/scripts/lgtm/dashboards/libtmux-transports.json index 50c6de033..1b89da99e 100644 --- a/scripts/lgtm/dashboards/libtmux-transports.json +++ b/scripts/lgtm/dashboards/libtmux-transports.json @@ -11,7 +11,7 @@ "version": 1, "editable": true, "graphTooltip": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-1h", "to": "now" diff --git a/scripts/lgtm/generate_dashboards.py b/scripts/lgtm/generate_dashboards.py index 98bbd9398..50a569086 100644 --- a/scripts/lgtm/generate_dashboards.py +++ b/scripts/lgtm/generate_dashboards.py @@ -140,7 +140,7 @@ def __init__( title: str, *, description: str = "", - refresh: str = "30s", + refresh: str = "1m", time_from: str = "now-1h", ) -> None: self.uid = uid @@ -187,10 +187,19 @@ def row(self, title: str) -> None: ) self._y += 1 - def add(self, panel: dict[str, t.Any], *, w: int = 12, h: int = 8) -> None: + def add( + self, + panel: dict[str, t.Any], + *, + w: int = 12, + h: int = 8, + links: list[dict[str, t.Any]] | None = None, + ) -> None: """Place a panel at the current grid cursor.""" panel["id"] = self._next_id() panel["gridPos"] = self._place(w, h) + if links: + panel["links"] = links self._panels.append(panel) def variable(self, name: str, label: str, metric_label: str) -> None: @@ -401,6 +410,38 @@ def table( } +def text(title: str, markdown: str) -> dict[str, t.Any]: + """Build a documentation panel. + + Grafana's guidance is that a dashboard should answer a question, and that + the question should be written down rather than inferred from the panels. + """ + return { + "type": "text", + "title": title, + "datasource": None, + "options": {"mode": "markdown", "content": markdown}, + "transparent": True, + } + + +def drill(title: str, uid: str) -> dict[str, t.Any]: + """Build a panel link that opens *uid* keeping time range and variables. + + Directed browsing is what separates a set of dashboards from a pile of + them: from a symptom, one click reaches the board that explains it, still + scoped to what you were looking at. + """ + return { + "title": title, + "url": ( + f"/d/{uid}?$" + "{__url_time_range}" + "&var-lane=${lane:queryparam}&var-branch=${branch:queryparam}" + ), + "targetBlank": False, + } + + def logs(title: str, expr: str, *, description: str = "") -> dict[str, t.Any]: """Build a Loki logs panel.""" return { @@ -476,7 +517,23 @@ def build_overview() -> Board: ) board.scope_variables() - board.row("Totals") + board.add( + text( + "", + "**Rate, Errors, Duration** for the tmux engines. RED is the right " + "frame here because an engine is a service: the caller cares how " + "much work goes through, how much of it fails, and how long it " + "takes.\n\n" + "Panels link to the board that explains them. Use **Transports** " + "for which transport is responsible, **Commands** for which tmux " + "command, and **Compare** for whether a branch or run changed " + "anything.", + ), + w=24, + h=4, + ) + + board.row("Rate") board.add( stat( "Requests", @@ -495,6 +552,7 @@ def build_overview() -> Board: w=6, h=5, ) + board.row("Errors") board.add( stat( "Inlined share", @@ -529,7 +587,7 @@ def build_overview() -> Board: h=5, ) - board.row("Throughput and latency") + board.row("Rate over time") board.add( timeseries( "Request rate by transport", @@ -541,7 +599,8 @@ def build_overview() -> Board: ], unit="reqps", description="How much each transport is carrying.", - ) + ), + links=[drill("Break down by transport", "libtmux-transports")], ) board.add( timeseries( @@ -578,7 +637,8 @@ def build_overview() -> Board: ], unit="reqps", description="tmux rejections; the smoke workload issues these on purpose.", - ) + ), + links=[drill("Break down by command", "libtmux-commands")], ) board.row("Traces, logs, and profiles") @@ -754,7 +814,6 @@ def build_commands() -> Board: ) ], unit="reqps", - stacking=True, ), w=16, h=9, @@ -971,7 +1030,94 @@ def quantile(quantile_value: float, label: str) -> str: return board -BUILDERS = (build_overview, build_transports, build_commands, build_compare) +def build_home() -> Board: + """Build the entry point that says which board answers which question. + + Grafana's maturity model calls this directed browsing: without it, finding + the right dashboard is guesswork and the answer is to duplicate one. + """ + board = Board( + "libtmux-home", + "libtmux / Home", + description="Start here. Which board answers which question.", + refresh="", + ) + board.scope_variables() + board.add( + text( + "", + "# libtmux engine observability\n\n" + "Telemetry comes from the engine instrumentation seam, so the " + "exporters are ordinary sinks and the engines are untouched.\n\n" + "| Board | Answers |\n" + "| --- | --- |\n" + "| [Overview](/d/libtmux-overview) | Is work flowing, and is it healthy? " + "Rate, errors, duration, plus traces, logs and profiles. |\n" + "| [Transports](/d/libtmux-transports) | Which transport is responsible? " + "Subprocess against control mode, sync against async. |\n" + "| [Commands](/d/libtmux-commands) | Which tmux command is responsible? |\n" + "| [Compare](/d/libtmux-compare) | " + "Did this run or branch change anything? |\n\n" + "**Producing data**: `just otel-smoke` for a steady workload, " + "`just otel-load` for a ramping arrival rate that exposes " + "saturation. `just otel-acceptance` checks every panel still has " + "data.\n\n" + "Every board is generated by `scripts/lgtm/generate_dashboards.py`; " + "edits made in this UI are overwritten on the next `just otel-up`.", + ), + w=24, + h=13, + ) + board.row("Health right now") + board.add( + stat( + "Requests in range", + [target(window_total("tmux_requests_total"), instant=True)], + description="Across every transport in the selected window.", + ), + w=8, + h=5, + links=[drill("Open Overview", "libtmux-overview")], + ) + board.add( + stat( + "Failure share", + [ + target( + f"100 * {window_total('tmux_failures_total')} " + f"/ clamp_min({window_total('tmux_requests_total')}, 1)", + instant=True, + ) + ], + unit="percent", + thresholds=ERR_THRESHOLDS, + description="Requests tmux rejected.", + ), + w=8, + h=5, + links=[drill("Open Commands", "libtmux-commands")], + ) + board.add( + stat( + "Branches reporting", + [ + target( + "count(count by (vcs_ref_head_name) (" + + window_total("tmux_requests_total", "vcs_ref_head_name") + + "))", + instant=True, + ) + ], + description="How many branches have data in this window.", + ), + w=8, + h=5, + links=[drill("Open Compare", "libtmux-compare")], + ) + return board + + +BUILDERS = (build_home, build_overview, build_transports, build_commands, build_compare) def write_dashboards(out_dir: pathlib.Path) -> list[pathlib.Path]: diff --git a/scripts/lgtm/load_tmux.py b/scripts/lgtm/load_tmux.py new file mode 100644 index 000000000..4c5a1b8c9 --- /dev/null +++ b/scripts/lgtm/load_tmux.py @@ -0,0 +1,183 @@ +"""Load-shape the tmux engines with rampa, and export the result to LGTM. + +``otel_smoke.py`` answers "does telemetry flow" by running flat out for a fixed +duration. That is the wrong shape for asking where a transport stops keeping +up, because a closed loop of N workers slows down with the system: offered load +falls as latency rises, and the graph bends politely instead of breaking. + +rampa supplies the shapes that expose that -- notably ``ramping-arrival-rate``, +an open model that keeps issuing commands at a target rate whether or not the +previous ones finished. Latency then climbs on its own when a transport +saturates, which is the knee worth finding. + +Run it through ``just otel-load``. +""" + +from __future__ import annotations + +import asyncio +import os +import pathlib +import shutil +import subprocess +import sys +import typing as t +import uuid + +# rampa loads this file by path, so its directory is not guaranteed to be +# importable; telemetry.py sits beside it. +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +import rampa +import telemetry + +from libtmux.experimental.engines import ( + AsyncControlModeEngine, + AsyncSubprocessEngine, + instrument, +) +from libtmux.experimental.engines.base import CommandRequest, CommandSeparator +from libtmux.experimental.engines.control_mode import command_count +from libtmux.server import Server + +LANE = os.environ.get("LIBTMUX_LOAD_LANE", "control-async") +PLAIN = CommandRequest.from_args("list-panes", "-a", "-F", "#{pane_id}") +GROUPED = CommandRequest.from_args( + "set-option", + "-g", + "@load", + "1", + CommandSeparator(";"), + "show-options", + "-g", + "@load", +) + +# One tmux server and one engine per process, shared by every virtual user. +# Building them per iteration would measure process startup rather than the +# transport, which is the opposite of the point. +_STATE: dict[str, t.Any] = {} + + +def _server() -> tuple[Server, pathlib.Path]: + """Create the throwaway tmux server this run drives.""" + root = pathlib.Path(f"/tmp/libtmux-load-{uuid.uuid4().hex[:8]}") + root.mkdir(mode=0o700) + socket_path = root / "load.sock" + subprocess.run( + ( + "tmux", + "-S", + str(socket_path), + "-f", + "/dev/null", + "new-session", + "-d", + "-s", + "load", + "sleep 600", + ), + check=True, + ) + subprocess.run( + ( + "tmux", + "-S", + str(socket_path), + "set-option", + "-g", + "destroy-unattached", + "off", + ), + check=True, + ) + return Server(socket_path=socket_path, config_file=os.devnull), root + + +def _engine() -> t.Any: + """Return the shared instrumented engine, building it on first use. + + Telemetry rides on this project's own sink rather than a rampa output + backend. rampa decides *when* each command is issued; the sink decides what + is recorded about it. Keeping that split means the load-shaped run lands in + Grafana under the same metric names and the same branch, worktree, and + spike labels as every other run, so the two are directly comparable instead + of arriving as a second, parallel account of the same work. + """ + if "engine" not in _STATE: + os.environ.pop("TMUX", None) + os.environ.pop("TMUX_PANE", None) + server, root = _server() + _STATE["root"] = root + signals = telemetry.build( + os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:4318"), + run_id=os.environ.get("LIBTMUX_RUN_ID", f"load-{uuid.uuid4().hex[:8]}"), + spike=os.environ.get("LIBTMUX_SPIKE"), + ) + _STATE["signals"] = signals + factory = { + "control-async": AsyncControlModeEngine.for_server, + "subprocess-async": AsyncSubprocessEngine.for_server, + }[LANE] + _STATE["engine"] = instrument( + factory(server), + telemetry.OTelSink( + signals.tracer, signals.meter, LANE, signals.metric_labels + ), + ) + return _STATE["engine"] + + +async def _issue(worker: rampa.Worker, request: CommandRequest) -> None: + """Run one request and record it under rampa's metric vocabulary.""" + engine = _engine() + tags = {"tmux_lane": LANE, "tmux_command": str(request.args[0])} + started = asyncio.get_running_loop().time() + result = await engine.run(request) + elapsed_ms = (asyncio.get_running_loop().time() - started) * 1000 + + worker.trend("tmux_command_duration", elapsed_ms, tags) + worker.counter("tmux_requests", 1.0, tags) + worker.counter("tmux_commands", float(command_count(tuple(request.args))), tags) + worker.check(result, {"tmux accepted": lambda r: r.returncode == 0}) + + +@rampa.scenario(executor="constant-vus", vus=8, duration="10s") +async def steady(worker: rampa.Worker) -> None: + """Hold a fixed concurrency, the closed-loop baseline.""" + await _issue(worker, PLAIN) + await _issue(worker, GROUPED) + + +@rampa.scenario( + executor="ramping-arrival-rate", + stages=[ + rampa.Stage(duration="8s", target=200), + rampa.Stage(duration="8s", target=1200), + ], + pre_allocated_vus=32, + max_vus=256, +) +async def ramp(worker: rampa.Worker) -> None: + """Raise offered rate regardless of latency, so saturation shows itself.""" + await _issue(worker, PLAIN) + + +async def teardown(_: t.Any = None) -> None: + """Close the engine and remove the tmux server.""" + engine = _STATE.get("engine") + if engine is not None and hasattr(engine.inner, "aclose"): + await engine.inner.aclose() + signals = _STATE.get("signals") + if signals is not None: + signals.shutdown() + root = _STATE.get("root") + if root is not None: + # Blocking work belongs off the loop, even during teardown. + await asyncio.to_thread( + subprocess.run, + ("tmux", "-S", str(root / "load.sock"), "kill-server"), + capture_output=True, + check=False, + ) + await asyncio.to_thread(shutil.rmtree, root, ignore_errors=True) diff --git a/tests/test_lgtm_dashboards.py b/tests/test_lgtm_dashboards.py index b4bb817fc..71008ecdf 100644 --- a/tests/test_lgtm_dashboards.py +++ b/tests/test_lgtm_dashboards.py @@ -51,6 +51,16 @@ def _panels(board: dict[str, t.Any]) -> list[dict[str, t.Any]]: return [panel for panel in board["panels"] if panel["type"] != "row"] +def _query_panels(board: dict[str, t.Any]) -> list[dict[str, t.Any]]: + """Return only the panels that are supposed to query a datasource. + + Text panels are documentation. Grafana's guidance is to write the question + a board answers onto the board itself, so a panel with no query is + expected here rather than a defect. + """ + return [panel for panel in _panels(board) if panel["type"] != "text"] + + def test_committed_dashboards_match_their_generator(tmp_path: pathlib.Path) -> None: """The JSON in the repo is what the generator produces right now. @@ -77,7 +87,7 @@ def test_committed_dashboards_match_their_generator(tmp_path: pathlib.Path) -> N def test_every_panel_queries_something() -> None: """A panel with no target can never show data.""" for board in _boards(): - for panel in _panels(board): + for panel in _query_panels(board): assert panel.get("targets"), ( f"{board['uid']}/{panel['title']} has no targets" ) @@ -86,7 +96,7 @@ def test_every_panel_queries_something() -> None: def test_every_target_binds_a_provisioned_datasource() -> None: """Panels reference datasources by uid, so the uid has to exist.""" for board in _boards(): - for panel in _panels(board): + for panel in _query_panels(board): for target in panel["targets"]: uid = (target.get("datasource") or {}).get("uid") assert uid in _DATASOURCE_UIDS, ( @@ -130,7 +140,7 @@ def test_acceptance_expands_every_template_variable() -> None: spec.loader.exec_module(acceptance) for board in _boards(): - for panel in _panels(board): + for panel in _query_panels(board): for target in panel["targets"]: query = ( target.get("expr") @@ -167,7 +177,7 @@ def test_no_panel_reads_a_counter_without_a_window() -> None: the series to have gone stale, which is exactly when nobody is looking. """ for board in _boards(): - for panel in _panels(board): + for panel in _query_panels(board): for target in panel["targets"]: if (target.get("datasource") or {}).get("type") != "prometheus": continue diff --git a/uv.lock b/uv.lock index ac50bb7c6..a9724a918 100644 --- a/uv.lock +++ b/uv.lock @@ -79,6 +79,162 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/cd/0d76dfc5de72bde52f55f53e925c7d152d9c7906634ec1e0cbc7e8d4ad93/aiofile-3.11.1-py3-none-any.whl", hash = "sha256:ce77d14ac07f77bc2b757834a5c129321f3f705c474593deed5ab209079a52c9", size = 20446, upload-time = "2026-05-16T08:18:32.051Z" }, ] +[[package]] +name = "aiohappyeyeballs" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/4d/4a99fb425c5e0cad715eea7bd190aff46f38b959a0a2dadb993705d34b26/aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b", size = 765848, upload-time = "2026-07-23T01:52:08.217Z" }, + { url = "https://files.pythonhosted.org/packages/74/e8/43b85dc55b8e950dc644babe762add781319ea881b57b33d2cce12017d12/aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a", size = 517476, upload-time = "2026-07-23T01:52:10.846Z" }, + { url = "https://files.pythonhosted.org/packages/7f/9e/73b582c4dbbc3c12ef4473822475effaabf1f934b56f14f5b03fe5d3a2af/aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5", size = 515334, upload-time = "2026-07-23T01:52:12.636Z" }, + { url = "https://files.pythonhosted.org/packages/79/03/e98c3c9e05a5bdf97defe5ff9169baba4f0ec9a901f2d60e0f060c2f051e/aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f", size = 1708830, upload-time = "2026-07-23T01:52:14.538Z" }, + { url = "https://files.pythonhosted.org/packages/d7/2c/26e60b694844dfd2176c57f913a22d0cd6a16f9ff202cbda7580d0328b98/aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43", size = 1674012, upload-time = "2026-07-23T01:52:16.486Z" }, + { url = "https://files.pythonhosted.org/packages/38/65/672df92e3172cd876aacfa97a952ac560877eb169384b2991ac5b273de4c/aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9", size = 1767015, upload-time = "2026-07-23T01:52:18.28Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c5/228dec7bfec1c373cc2217cdeb47d6456dcd7a13a4c55144930a75ae3851/aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8", size = 1858700, upload-time = "2026-07-23T01:52:20.08Z" }, + { url = "https://files.pythonhosted.org/packages/bd/ff/cb36724e8c8d17f90ada567a9ff3efe1d6e9b549fba697a242aece180f21/aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479", size = 1714075, upload-time = "2026-07-23T01:52:22.071Z" }, + { url = "https://files.pythonhosted.org/packages/9f/3a/296a4135c6366376263aeef54b15caca1f07676c2ae0c525d7832f2f808a/aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b", size = 1588234, upload-time = "2026-07-23T01:52:23.757Z" }, + { url = "https://files.pythonhosted.org/packages/7d/81/9d5d853ef892dc066d1eb6db0e87a47348b920c1c879aa554612fdbd9d79/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d", size = 1677300, upload-time = "2026-07-23T01:52:25.861Z" }, + { url = "https://files.pythonhosted.org/packages/68/96/021d386ae32d9b26d4b88df2e794546232ff56bb6be952bf6be227c0bbc7/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d", size = 1691501, upload-time = "2026-07-23T01:52:28Z" }, + { url = "https://files.pythonhosted.org/packages/29/9f/af66adce26a14af135c003cbd0f44ccaa68cebd30ff8ac99ca47fb4958f7/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2", size = 1735113, upload-time = "2026-07-23T01:52:29.995Z" }, + { url = "https://files.pythonhosted.org/packages/2f/90/28c390d4c9851effe52ac25b5a2e1d92246acd00728b4fc7975dafb67484/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48", size = 1577486, upload-time = "2026-07-23T01:52:31.937Z" }, + { url = "https://files.pythonhosted.org/packages/db/c2/00e23a1bf2abb70dd353f6987db7e7f2491d0261f7363997738c71c98f95/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f", size = 1751353, upload-time = "2026-07-23T01:52:33.688Z" }, + { url = "https://files.pythonhosted.org/packages/6e/7d/d51a706a8cbfa57f0611127daf61ab3ae02ab8420b0407412079227d1c65/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32", size = 1698681, upload-time = "2026-07-23T01:52:38.167Z" }, + { url = "https://files.pythonhosted.org/packages/ec/b0/90bd5cd9fdd9787cb4211d284d1fb8401339a933cb0227a15b71e789232f/aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e", size = 456733, upload-time = "2026-07-23T01:52:41.823Z" }, + { url = "https://files.pythonhosted.org/packages/d8/15/fe5b8f6a71ae112bc677163d0b0701bda5dc15005249582258ede0eb88c7/aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c", size = 480460, upload-time = "2026-07-23T01:52:43.905Z" }, + { url = "https://files.pythonhosted.org/packages/54/00/45e98b6645cd7f00a4b78b749ebd309094b0eaeb2d2e96157eadbc0d0050/aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb", size = 453479, upload-time = "2026-07-23T01:52:46.075Z" }, + { url = "https://files.pythonhosted.org/packages/f8/5c/b3e4ff8ad43a8afef9602c5e90285936da1beaea8b029016b793891f03c3/aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3", size = 764250, upload-time = "2026-07-23T01:52:48.525Z" }, + { url = "https://files.pythonhosted.org/packages/0e/da/f1b384465e51449d844056b75070461da03a9a23e6c1747003695bf4172a/aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a", size = 516281, upload-time = "2026-07-23T01:52:51.047Z" }, + { url = "https://files.pythonhosted.org/packages/b9/3f/01264f820ee2e3712a827892b1cd6ff80f3300c1fcbffbb45714a915d47a/aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8", size = 514742, upload-time = "2026-07-23T01:52:53.779Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8d/a71c6f2db52ac1ed142b133f7feddaa6b70539c3f4de24d7e226c95b794c/aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239", size = 1780613, upload-time = "2026-07-23T01:52:56.948Z" }, + { url = "https://files.pythonhosted.org/packages/a5/11/3dd9b3fb3a170f6ec9011b5291d876a6fab4086714c9e158600edf01b4fd/aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f", size = 1737688, upload-time = "2026-07-23T01:52:59.294Z" }, + { url = "https://files.pythonhosted.org/packages/6d/3e/834c26918be7d88068822b40e0db30fca50b5f4fe79104aa16a93f1d74e6/aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06", size = 1845742, upload-time = "2026-07-23T01:53:01.641Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c9/49ab8572df7d66bc13d11e31f781292badb04180dd87ba98733066c6aed7/aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929", size = 1928412, upload-time = "2026-07-23T01:53:04.018Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b9/2b8f0c0ce09c87a1daf80fd483431b56b1435d3f62789bc86f572e1245de/aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db", size = 1786220, upload-time = "2026-07-23T01:53:06.481Z" }, + { url = "https://files.pythonhosted.org/packages/85/00/9c45f81de11710460edfa1dc81317b6e882703b160926c879a9d20da9fcc/aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce", size = 1637231, upload-time = "2026-07-23T01:53:10.258Z" }, + { url = "https://files.pythonhosted.org/packages/19/ce/967d628e910756f3539c6107cb7844a1b69440dcb3029a5ee7871b09ab63/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c", size = 1753161, upload-time = "2026-07-23T01:53:13.817Z" }, + { url = "https://files.pythonhosted.org/packages/11/b2/0c3d4114f0aee4f580f5b3b4eb71b24d7a23b834ea506a4dfebe76513f35/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15", size = 1756356, upload-time = "2026-07-23T01:53:16.211Z" }, + { url = "https://files.pythonhosted.org/packages/63/5d/99e7d91c82f1399d1ae2a854e080bd1493fbc31e5e959dbc4ec33dac3bec/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c", size = 1819846, upload-time = "2026-07-23T01:53:18.289Z" }, + { url = "https://files.pythonhosted.org/packages/ad/05/d5e1cb6480eeffd3f901d40a2c5e2d1e7effdc797837da3b490272699f13/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae", size = 1628531, upload-time = "2026-07-23T01:53:23.86Z" }, + { url = "https://files.pythonhosted.org/packages/c9/90/b934682bcaefae18a9e04f3dff5b68522ba810906358ae5029b68110ea3b/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910", size = 1832712, upload-time = "2026-07-23T01:53:27.551Z" }, + { url = "https://files.pythonhosted.org/packages/21/df/6061679faaf81fac746e7307c7adb71e858071a5d34c27583afefc64f543/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7", size = 1775014, upload-time = "2026-07-23T01:53:30.223Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1d/f854878bbc69b88faefe924b619a34a6f59ec05fd387c77690667eaa75eb/aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa", size = 456006, upload-time = "2026-07-23T01:53:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/73/0c/2af9d1674baccd1dbd47282a93d660a22e57ef6167c856deb24b4214fbab/aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d", size = 481069, upload-time = "2026-07-23T01:53:39.673Z" }, + { url = "https://files.pythonhosted.org/packages/8e/76/88401ff3fc95e85c5fc38d588f36f55e61ecb64343b2bc8d69326f453cc0/aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39", size = 453021, upload-time = "2026-07-23T01:53:43.749Z" }, + { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" }, + { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" }, + { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" }, + { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" }, + { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" }, + { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" }, + { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" }, + { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" }, + { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" }, + { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" }, + { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" }, + { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" }, + { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" }, + { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" }, + { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" }, + { url = "https://files.pythonhosted.org/packages/57/be/5afd201cc0ab139029aadb75392efe85a293403d9dd3a3226161c21ce00c/aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86", size = 506269, upload-time = "2026-07-23T01:54:49.075Z" }, + { url = "https://files.pythonhosted.org/packages/22/09/dec8189d62b45ade009f6792a2264b942a90cb88aeaf181239933cd72c3c/aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627", size = 515166, upload-time = "2026-07-23T01:54:51.894Z" }, + { url = "https://files.pythonhosted.org/packages/28/24/2854869d29ed8a8b19d74f9ec6629515f7e04d02dd329d9d179201e58e47/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82", size = 486263, upload-time = "2026-07-23T01:54:54.223Z" }, + { url = "https://files.pythonhosted.org/packages/d4/dd/57187c8be2a35aea65eaee3bd2c3dcbbcf0204f5106c89637e3610380cd1/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c", size = 492299, upload-time = "2026-07-23T01:54:56.236Z" }, + { url = "https://files.pythonhosted.org/packages/b9/11/06ae6ed8f0d414edf4068861e233d8fe23ee699bfd4b3ceb8663db948a62/aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f", size = 502235, upload-time = "2026-07-23T01:54:58.377Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a3/559639c34a345d2cf7c52dff6838119f2eaf29eb508227b5b83f573af813/aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80", size = 750883, upload-time = "2026-07-23T01:55:00.65Z" }, + { url = "https://files.pythonhosted.org/packages/91/cd/41e131f13afd1e7b0172a9d9eda085ef90eb8439f41f0d279db81ed3ae60/aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0", size = 508473, upload-time = "2026-07-23T01:55:02.945Z" }, + { url = "https://files.pythonhosted.org/packages/bc/6b/e7f13410d391c6e55b4c007a8de024355389d7d459e3d64c42b2d33617e5/aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf", size = 509190, upload-time = "2026-07-23T01:55:05.173Z" }, + { url = "https://files.pythonhosted.org/packages/97/21/6464573e53d69672cc1eada3e5c5cb2d2efa82701e8305a0f2047a576967/aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd", size = 1761478, upload-time = "2026-07-23T01:55:07.383Z" }, + { url = "https://files.pythonhosted.org/packages/1a/81/d217043a4c17fbce360905e3b2bdd20139ebc9a2de836d035d179c4da006/aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807", size = 1735092, upload-time = "2026-07-23T01:55:09.803Z" }, + { url = "https://files.pythonhosted.org/packages/a1/66/e13a02d0eeb1a9a502402a977abb4e4abff9fe4051c26f80558c57a7c975/aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8", size = 1800546, upload-time = "2026-07-23T01:55:12.012Z" }, + { url = "https://files.pythonhosted.org/packages/26/5e/57d42fca1d18cb5acc1cad945d017fabc5d6ae71d8a08ad66be8dc3ee544/aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24", size = 1895250, upload-time = "2026-07-23T01:55:14.357Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1c/7da8d08e74d56f00070822f9638ff3f1c563f8ad87d1efa996c87bfc8644/aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5", size = 1789289, upload-time = "2026-07-23T01:55:16.668Z" }, + { url = "https://files.pythonhosted.org/packages/cd/0f/cf16bcf56896981c1a0319f5d5db9337994b5165730c48a8fa07e9b34be6/aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4", size = 1586706, upload-time = "2026-07-23T01:55:18.913Z" }, + { url = "https://files.pythonhosted.org/packages/fe/6f/76eac12a7f2480e1e304f842efdb07db33256b0d9165b866b6ef0806c202/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9", size = 1724652, upload-time = "2026-07-23T01:55:21.296Z" }, + { url = "https://files.pythonhosted.org/packages/39/b6/19c8c592baeeb94b75f966547d40c02ac7590902306ec5863d5c027cf506/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1", size = 1756239, upload-time = "2026-07-23T01:55:23.705Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c9/4e9383150296f97f873b680c4de8fb2cd88608fb9f48c79edcb111611abc/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371", size = 1769161, upload-time = "2026-07-23T01:55:26.082Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1e/147bdc6cc5de5f3ab011be8bf5d6e786633249f22c20bae06f85e45f5387/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde", size = 1578759, upload-time = "2026-07-23T01:55:28.846Z" }, + { url = "https://files.pythonhosted.org/packages/fd/31/78388a9d6040ece2e11df62ea229a822cf5e52d238374b220ae9975b2623/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e", size = 1792025, upload-time = "2026-07-23T01:55:31.457Z" }, + { url = "https://files.pythonhosted.org/packages/03/51/a3d29fdf2c25d796746af8ad6fe56a45d6256c38b0a8a2ed752e1160b3a2/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71", size = 1768477, upload-time = "2026-07-23T01:55:33.87Z" }, + { url = "https://files.pythonhosted.org/packages/29/a6/442e18b5afeade534d877a2dc3c3e392aff8d49787890b0cf84790410267/aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0", size = 451069, upload-time = "2026-07-23T01:55:36.121Z" }, + { url = "https://files.pythonhosted.org/packages/9d/69/3d876ac02659f271cf7f6769f14a8e3de5b6e888ed8b5a7e998086a4cec8/aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883", size = 476518, upload-time = "2026-07-23T01:55:38.303Z" }, + { url = "https://files.pythonhosted.org/packages/b2/0e/50d6e6471cd31edce8b282bdec59375a3a69124d8a989a0b1313355cae52/aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2", size = 447676, upload-time = "2026-07-23T01:55:40.451Z" }, + { url = "https://files.pythonhosted.org/packages/c8/20/887fdcf832326571b370ffc347b3e70abe101096f3720126aac161b1d872/aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062", size = 509067, upload-time = "2026-07-23T01:55:42.618Z" }, + { url = "https://files.pythonhosted.org/packages/ad/a3/92cec936f78cc4bf0fa5554ebe593b73459d94e3c62303e1902a4cccb6f7/aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6", size = 514774, upload-time = "2026-07-23T01:55:44.937Z" }, + { url = "https://files.pythonhosted.org/packages/29/ba/2a0c38df3fc557620b6a5acd98364af050053b6285b4dc7ee74100c63c18/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919", size = 488134, upload-time = "2026-07-23T01:55:47.135Z" }, + { url = "https://files.pythonhosted.org/packages/48/d6/d51b7d4bf309af3693940d8ffd2b9ed0b682434ef85959b7c9c137f60cf8/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7", size = 494201, upload-time = "2026-07-23T01:55:49.451Z" }, + { url = "https://files.pythonhosted.org/packages/3f/5a/8f624384e5f1efabb5229b94157eb966b021e97bdb188c62860c2ae243c2/aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0", size = 502766, upload-time = "2026-07-23T01:55:51.656Z" }, + { url = "https://files.pythonhosted.org/packages/a6/26/4ff0164370deec18fb19254ee4ab10b7a73304ac0c860b13f5f84663759b/aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924", size = 756557, upload-time = "2026-07-23T01:55:53.964Z" }, + { url = "https://files.pythonhosted.org/packages/97/a3/7056b86dc0d9ec709ea9777eae3b0161428f943372f8b98c01c11593b682/aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646", size = 510168, upload-time = "2026-07-23T01:55:56.22Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/0357a015892fd68058bf2d39d3fd1958e459b997a7db30aaa6aaa434ae96/aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b", size = 512957, upload-time = "2026-07-23T01:55:58.437Z" }, + { url = "https://files.pythonhosted.org/packages/47/d1/8aba53f15ccb2238405f5e9d30e2a8ca44f93878c26e7165ade00d374b1c/aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30", size = 1750149, upload-time = "2026-07-23T01:56:00.856Z" }, + { url = "https://files.pythonhosted.org/packages/49/bd/40c3fee327529284375c6701cbb0fa4600cc2e8432af1378f897e2ef7d3a/aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9", size = 1707685, upload-time = "2026-07-23T01:56:03.371Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a3/ca0cc6724cca8114b05694abd916060758c79894c3aa5b012cdadc1bc28e/aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f", size = 1803911, upload-time = "2026-07-23T01:56:05.817Z" }, + { url = "https://files.pythonhosted.org/packages/95/b5/85b099c299c3ffd38ad9b3e43694c8a346934e4a30c88c4fd5a841234f77/aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d", size = 1876929, upload-time = "2026-07-23T01:56:08.413Z" }, + { url = "https://files.pythonhosted.org/packages/d5/b7/1da684a04175473fa4cddbf9a2f572e79514c3fd27a74597f43057d4f3da/aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147", size = 1761112, upload-time = "2026-07-23T01:56:10.918Z" }, + { url = "https://files.pythonhosted.org/packages/d1/16/bc4b55e3e5cb175fd69c53c90d60d2f47797cb343da5106e23863dc4dba4/aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c", size = 1583500, upload-time = "2026-07-23T01:56:13.613Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e8/13a9d957a1ee40837f46aa30f0f4c657e673ad86a2e6362a9f9be20d26d9/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a", size = 1713940, upload-time = "2026-07-23T01:56:15.969Z" }, + { url = "https://files.pythonhosted.org/packages/38/05/d33c680c1bcf1c7e130f9cbfc1fc02fe8bb0c4af2a94a53dd5fb56131e5c/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0", size = 1724413, upload-time = "2026-07-23T01:56:18.591Z" }, + { url = "https://files.pythonhosted.org/packages/85/1d/af798d306f7a74b6a632dbcabcf62a4c91391b7582d2a8c6d7712e2cc54e/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661", size = 1770748, upload-time = "2026-07-23T01:56:21.074Z" }, + { url = "https://files.pythonhosted.org/packages/a8/92/ad720d472556a995049206867765e9410969684f86ee09423ff9969044c1/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22", size = 1577564, upload-time = "2026-07-23T01:56:23.475Z" }, + { url = "https://files.pythonhosted.org/packages/60/ad/0ed7586cbef7a884e23a752fa2bb987a122e6a5dd50dab109258d0a95193/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41", size = 1782080, upload-time = "2026-07-23T01:56:25.994Z" }, + { url = "https://files.pythonhosted.org/packages/97/ea/dbaed0d73e8a69aad653b045dab451c67c2454bb731a37b45a86593e9422/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf", size = 1745813, upload-time = "2026-07-23T01:56:28.604Z" }, + { url = "https://files.pythonhosted.org/packages/81/1b/6893d4bc57e434fc93a6c9217c637d967a0b651d989f6e3265179375754a/aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da", size = 455872, upload-time = "2026-07-23T01:56:31.031Z" }, + { url = "https://files.pythonhosted.org/packages/f5/8b/c7baa1ba1eda4db6989baefe5de6d99834921b84ebd7918624febcb9f290/aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100", size = 481030, upload-time = "2026-07-23T01:56:33.365Z" }, + { url = "https://files.pythonhosted.org/packages/22/8c/c29d067df825a2df88ca432db848aa2fe8199598359cc06c12b09320cac9/aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc", size = 453669, upload-time = "2026-07-23T01:56:35.731Z" }, + { url = "https://files.pythonhosted.org/packages/6a/a4/9c033beb355d39b6147980597ec9645e4729243f686ee4dc73945de72030/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b", size = 791403, upload-time = "2026-07-23T01:56:37.972Z" }, + { url = "https://files.pythonhosted.org/packages/80/ca/87c32a0a7704583cfc49660bd817889bae5b830bf53b5dcb4e92145ac2da/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0", size = 526413, upload-time = "2026-07-23T01:56:40.523Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d8/8ec0e471248c500acdce2be3f46db8fb62b5eb60efef072529cc85ee1d26/aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e", size = 532135, upload-time = "2026-07-23T01:56:42.876Z" }, + { url = "https://files.pythonhosted.org/packages/fe/45/f8919fd936e8b79fcd9bda7b6d8e62613462a713f4f17987fd7c34399142/aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716", size = 1922742, upload-time = "2026-07-23T01:56:45.528Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ec/9ca76b28a27525b0cc53e20842e0228b022f301ce1f436b7d814b4aaf2df/aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f", size = 1787371, upload-time = "2026-07-23T01:56:48.045Z" }, + { url = "https://files.pythonhosted.org/packages/b1/04/6acdbf17315f7b55f1937e3387acb89a3cddeb4995689553d064af8e92ab/aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553", size = 1912623, upload-time = "2026-07-23T01:56:50.605Z" }, + { url = "https://files.pythonhosted.org/packages/86/e6/438b0c79ca6f45eb9fd9817dd4c01a91919a38c0de5ee9e05e2b4dc0ece7/aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100", size = 2005515, upload-time = "2026-07-23T01:56:53.153Z" }, + { url = "https://files.pythonhosted.org/packages/bb/6b/62cbd6577758699525f5c712d1ddef57d9875fbab0ae8d5f5a202fd598f8/aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85", size = 1879906, upload-time = "2026-07-23T01:56:55.818Z" }, + { url = "https://files.pythonhosted.org/packages/00/95/18bcbf830a21dc3aae24d8f6b6feaf3db1d2090242d00a7868db2ffb0b67/aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33", size = 1675849, upload-time = "2026-07-23T01:56:58.861Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/47f4968659c5e23606c3790c80fc624e691c153d036148449ee84d31b287/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f", size = 1843496, upload-time = "2026-07-23T01:57:01.591Z" }, + { url = "https://files.pythonhosted.org/packages/64/af/38c33c4dd82fddcb4e56c4653b6f1072a8edbc6b7fa15809f14932c41e2d/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0", size = 1827746, upload-time = "2026-07-23T01:57:05.131Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9d/0537cda4885ac8f5b7053d164dd06312f4c483a4edcb8ee5b8aaf2a989bf/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098", size = 1853810, upload-time = "2026-07-23T01:57:08.043Z" }, + { url = "https://files.pythonhosted.org/packages/19/fe/26f9c5e6458385aa86497836b0dea6fb2f027827d63f37c7856cce9286ee/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25", size = 1668895, upload-time = "2026-07-23T01:57:10.837Z" }, + { url = "https://files.pythonhosted.org/packages/ec/4c/618b1db9b9ba079b8875d2cdf78e7c4a3bf72903bd5850fee7dd9544600a/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9", size = 1883833, upload-time = "2026-07-23T01:57:13.672Z" }, + { url = "https://files.pythonhosted.org/packages/94/c6/bd959bd1e4771f9fd944e9e436224c48c77b018b73b519b5aad346335bcc/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb", size = 1844251, upload-time = "2026-07-23T01:57:16.593Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/08d41839658bdd44a0ed2480f3891705ecb487ce28c0dde62c9040c997e0/aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963", size = 474180, upload-time = "2026-07-23T01:57:19.306Z" }, + { url = "https://files.pythonhosted.org/packages/99/5d/3cd6ef0a2b2851f7ab913b5b079334781bd50ff56a323e4454063377a080/aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b", size = 500528, upload-time = "2026-07-23T01:57:21.762Z" }, + { url = "https://files.pythonhosted.org/packages/a4/37/cfd1ed540a4d318da025590d96b728e63713c09e9377950fc655dadeb856/aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7", size = 469280, upload-time = "2026-07-23T01:57:24.241Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + [[package]] name = "alabaster" version = "1.0.0" @@ -765,6 +921,127 @@ server = [ { name = "websockets" }, ] +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/4a/557715d5047da48d54e659203b9335be7bfaafda2c3f627b7c47e0b3aaf3/frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011", size = 86230, upload-time = "2025-10-06T05:35:23.699Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fb/c85f9fed3ea8fe8740e5b46a59cc141c23b842eca617da8876cfce5f760e/frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565", size = 49621, upload-time = "2025-10-06T05:35:25.341Z" }, + { url = "https://files.pythonhosted.org/packages/63/70/26ca3f06aace16f2352796b08704338d74b6d1a24ca38f2771afbb7ed915/frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad", size = 49889, upload-time = "2025-10-06T05:35:26.797Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ed/c7895fd2fde7f3ee70d248175f9b6cdf792fb741ab92dc59cd9ef3bd241b/frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2", size = 219464, upload-time = "2025-10-06T05:35:28.254Z" }, + { url = "https://files.pythonhosted.org/packages/6b/83/4d587dccbfca74cb8b810472392ad62bfa100bf8108c7223eb4c4fa2f7b3/frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186", size = 221649, upload-time = "2025-10-06T05:35:29.454Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c6/fd3b9cd046ec5fff9dab66831083bc2077006a874a2d3d9247dea93ddf7e/frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e", size = 219188, upload-time = "2025-10-06T05:35:30.951Z" }, + { url = "https://files.pythonhosted.org/packages/ce/80/6693f55eb2e085fc8afb28cf611448fb5b90e98e068fa1d1b8d8e66e5c7d/frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450", size = 231748, upload-time = "2025-10-06T05:35:32.101Z" }, + { url = "https://files.pythonhosted.org/packages/97/d6/e9459f7c5183854abd989ba384fe0cc1a0fb795a83c033f0571ec5933ca4/frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef", size = 236351, upload-time = "2025-10-06T05:35:33.834Z" }, + { url = "https://files.pythonhosted.org/packages/97/92/24e97474b65c0262e9ecd076e826bfd1d3074adcc165a256e42e7b8a7249/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4", size = 218767, upload-time = "2025-10-06T05:35:35.205Z" }, + { url = "https://files.pythonhosted.org/packages/ee/bf/dc394a097508f15abff383c5108cb8ad880d1f64a725ed3b90d5c2fbf0bb/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff", size = 235887, upload-time = "2025-10-06T05:35:36.354Z" }, + { url = "https://files.pythonhosted.org/packages/40/90/25b201b9c015dbc999a5baf475a257010471a1fa8c200c843fd4abbee725/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c", size = 228785, upload-time = "2025-10-06T05:35:37.949Z" }, + { url = "https://files.pythonhosted.org/packages/84/f4/b5bc148df03082f05d2dd30c089e269acdbe251ac9a9cf4e727b2dbb8a3d/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f", size = 230312, upload-time = "2025-10-06T05:35:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/db/4b/87e95b5d15097c302430e647136b7d7ab2398a702390cf4c8601975709e7/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7", size = 217650, upload-time = "2025-10-06T05:35:40.377Z" }, + { url = "https://files.pythonhosted.org/packages/e5/70/78a0315d1fea97120591a83e0acd644da638c872f142fd72a6cebee825f3/frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a", size = 39659, upload-time = "2025-10-06T05:35:41.863Z" }, + { url = "https://files.pythonhosted.org/packages/66/aa/3f04523fb189a00e147e60c5b2205126118f216b0aa908035c45336e27e4/frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6", size = 43837, upload-time = "2025-10-06T05:35:43.205Z" }, + { url = "https://files.pythonhosted.org/packages/39/75/1135feecdd7c336938bd55b4dc3b0dfc46d85b9be12ef2628574b28de776/frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e", size = 39989, upload-time = "2025-10-06T05:35:44.596Z" }, + { url = "https://files.pythonhosted.org/packages/bc/03/077f869d540370db12165c0aa51640a873fb661d8b315d1d4d67b284d7ac/frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84", size = 86912, upload-time = "2025-10-06T05:35:45.98Z" }, + { url = "https://files.pythonhosted.org/packages/df/b5/7610b6bd13e4ae77b96ba85abea1c8cb249683217ef09ac9e0ae93f25a91/frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9", size = 50046, upload-time = "2025-10-06T05:35:47.009Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ef/0e8f1fe32f8a53dd26bdd1f9347efe0778b0fddf62789ea683f4cc7d787d/frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93", size = 50119, upload-time = "2025-10-06T05:35:48.38Z" }, + { url = "https://files.pythonhosted.org/packages/11/b1/71a477adc7c36e5fb628245dfbdea2166feae310757dea848d02bd0689fd/frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f", size = 231067, upload-time = "2025-10-06T05:35:49.97Z" }, + { url = "https://files.pythonhosted.org/packages/45/7e/afe40eca3a2dc19b9904c0f5d7edfe82b5304cb831391edec0ac04af94c2/frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695", size = 233160, upload-time = "2025-10-06T05:35:51.729Z" }, + { url = "https://files.pythonhosted.org/packages/a6/aa/7416eac95603ce428679d273255ffc7c998d4132cfae200103f164b108aa/frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52", size = 228544, upload-time = "2025-10-06T05:35:53.246Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3d/2a2d1f683d55ac7e3875e4263d28410063e738384d3adc294f5ff3d7105e/frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581", size = 243797, upload-time = "2025-10-06T05:35:54.497Z" }, + { url = "https://files.pythonhosted.org/packages/78/1e/2d5565b589e580c296d3bb54da08d206e797d941a83a6fdea42af23be79c/frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567", size = 247923, upload-time = "2025-10-06T05:35:55.861Z" }, + { url = "https://files.pythonhosted.org/packages/aa/c3/65872fcf1d326a7f101ad4d86285c403c87be7d832b7470b77f6d2ed5ddc/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b", size = 230886, upload-time = "2025-10-06T05:35:57.399Z" }, + { url = "https://files.pythonhosted.org/packages/a0/76/ac9ced601d62f6956f03cc794f9e04c81719509f85255abf96e2510f4265/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92", size = 245731, upload-time = "2025-10-06T05:35:58.563Z" }, + { url = "https://files.pythonhosted.org/packages/b9/49/ecccb5f2598daf0b4a1415497eba4c33c1e8ce07495eb07d2860c731b8d5/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d", size = 241544, upload-time = "2025-10-06T05:35:59.719Z" }, + { url = "https://files.pythonhosted.org/packages/53/4b/ddf24113323c0bbcc54cb38c8b8916f1da7165e07b8e24a717b4a12cbf10/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd", size = 241806, upload-time = "2025-10-06T05:36:00.959Z" }, + { url = "https://files.pythonhosted.org/packages/a7/fb/9b9a084d73c67175484ba2789a59f8eebebd0827d186a8102005ce41e1ba/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967", size = 229382, upload-time = "2025-10-06T05:36:02.22Z" }, + { url = "https://files.pythonhosted.org/packages/95/a3/c8fb25aac55bf5e12dae5c5aa6a98f85d436c1dc658f21c3ac73f9fa95e5/frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25", size = 39647, upload-time = "2025-10-06T05:36:03.409Z" }, + { url = "https://files.pythonhosted.org/packages/0a/f5/603d0d6a02cfd4c8f2a095a54672b3cf967ad688a60fb9faf04fc4887f65/frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b", size = 44064, upload-time = "2025-10-06T05:36:04.368Z" }, + { url = "https://files.pythonhosted.org/packages/5d/16/c2c9ab44e181f043a86f9a8f84d5124b62dbcb3a02c0977ec72b9ac1d3e0/frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a", size = 39937, upload-time = "2025-10-06T05:36:05.669Z" }, + { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, + { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, + { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, + { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, + { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, + { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, + { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, + { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, + { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, + { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, + { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, + { url = "https://files.pythonhosted.org/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", size = 85717, upload-time = "2025-10-06T05:36:27.341Z" }, + { url = "https://files.pythonhosted.org/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", size = 49651, upload-time = "2025-10-06T05:36:28.855Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", size = 49417, upload-time = "2025-10-06T05:36:29.877Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4e/e4691508f9477ce67da2015d8c00acd751e6287739123113a9fca6f1604e/frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", size = 234391, upload-time = "2025-10-06T05:36:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/40/76/c202df58e3acdf12969a7895fd6f3bc016c642e6726aa63bd3025e0fc71c/frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", size = 233048, upload-time = "2025-10-06T05:36:32.531Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c0/8746afb90f17b73ca5979c7a3958116e105ff796e718575175319b5bb4ce/frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", size = 226549, upload-time = "2025-10-06T05:36:33.706Z" }, + { url = "https://files.pythonhosted.org/packages/7e/eb/4c7eefc718ff72f9b6c4893291abaae5fbc0c82226a32dcd8ef4f7a5dbef/frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", size = 239833, upload-time = "2025-10-06T05:36:34.947Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/e5c02187cf704224f8b21bee886f3d713ca379535f16893233b9d672ea71/frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", size = 245363, upload-time = "2025-10-06T05:36:36.534Z" }, + { url = "https://files.pythonhosted.org/packages/1f/96/cb85ec608464472e82ad37a17f844889c36100eed57bea094518bf270692/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", size = 229314, upload-time = "2025-10-06T05:36:38.582Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6f/4ae69c550e4cee66b57887daeebe006fe985917c01d0fff9caab9883f6d0/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", size = 243365, upload-time = "2025-10-06T05:36:40.152Z" }, + { url = "https://files.pythonhosted.org/packages/7a/58/afd56de246cf11780a40a2c28dc7cbabbf06337cc8ddb1c780a2d97e88d8/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", size = 237763, upload-time = "2025-10-06T05:36:41.355Z" }, + { url = "https://files.pythonhosted.org/packages/cb/36/cdfaf6ed42e2644740d4a10452d8e97fa1c062e2a8006e4b09f1b5fd7d63/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", size = 240110, upload-time = "2025-10-06T05:36:42.716Z" }, + { url = "https://files.pythonhosted.org/packages/03/a8/9ea226fbefad669f11b52e864c55f0bd57d3c8d7eb07e9f2e9a0b39502e1/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", size = 233717, upload-time = "2025-10-06T05:36:44.251Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0b/1b5531611e83ba7d13ccc9988967ea1b51186af64c42b7a7af465dcc9568/frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", size = 39628, upload-time = "2025-10-06T05:36:45.423Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cf/174c91dbc9cc49bc7b7aab74d8b734e974d1faa8f191c74af9b7e80848e6/frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", size = 43882, upload-time = "2025-10-06T05:36:46.796Z" }, + { url = "https://files.pythonhosted.org/packages/c1/17/502cd212cbfa96eb1388614fe39a3fc9ab87dbbe042b66f97acb57474834/frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", size = 39676, upload-time = "2025-10-06T05:36:47.8Z" }, + { url = "https://files.pythonhosted.org/packages/d2/5c/3bbfaa920dfab09e76946a5d2833a7cbdf7b9b4a91c714666ac4855b88b4/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", size = 89235, upload-time = "2025-10-06T05:36:48.78Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d6/f03961ef72166cec1687e84e8925838442b615bd0b8854b54923ce5b7b8a/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", size = 50742, upload-time = "2025-10-06T05:36:49.837Z" }, + { url = "https://files.pythonhosted.org/packages/1e/bb/a6d12b7ba4c3337667d0e421f7181c82dda448ce4e7ad7ecd249a16fa806/frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", size = 51725, upload-time = "2025-10-06T05:36:50.851Z" }, + { url = "https://files.pythonhosted.org/packages/bc/71/d1fed0ffe2c2ccd70b43714c6cab0f4188f09f8a67a7914a6b46ee30f274/frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", size = 284533, upload-time = "2025-10-06T05:36:51.898Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/fb1685a7b009d89f9bf78a42d94461bc06581f6e718c39344754a5d9bada/frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", size = 292506, upload-time = "2025-10-06T05:36:53.101Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3b/b991fe1612703f7e0d05c0cf734c1b77aaf7c7d321df4572e8d36e7048c8/frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", size = 274161, upload-time = "2025-10-06T05:36:54.309Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ec/c5c618767bcdf66e88945ec0157d7f6c4a1322f1473392319b7a2501ded7/frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", size = 294676, upload-time = "2025-10-06T05:36:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ce/3934758637d8f8a88d11f0585d6495ef54b2044ed6ec84492a91fa3b27aa/frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", size = 300638, upload-time = "2025-10-06T05:36:56.758Z" }, + { url = "https://files.pythonhosted.org/packages/fc/4f/a7e4d0d467298f42de4b41cbc7ddaf19d3cfeabaf9ff97c20c6c7ee409f9/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", size = 283067, upload-time = "2025-10-06T05:36:57.965Z" }, + { url = "https://files.pythonhosted.org/packages/dc/48/c7b163063d55a83772b268e6d1affb960771b0e203b632cfe09522d67ea5/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", size = 292101, upload-time = "2025-10-06T05:36:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d0/2366d3c4ecdc2fd391e0afa6e11500bfba0ea772764d631bbf82f0136c9d/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", size = 289901, upload-time = "2025-10-06T05:37:00.811Z" }, + { url = "https://files.pythonhosted.org/packages/b8/94/daff920e82c1b70e3618a2ac39fbc01ae3e2ff6124e80739ce5d71c9b920/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", size = 289395, upload-time = "2025-10-06T05:37:02.115Z" }, + { url = "https://files.pythonhosted.org/packages/e3/20/bba307ab4235a09fdcd3cc5508dbabd17c4634a1af4b96e0f69bfe551ebd/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", size = 283659, upload-time = "2025-10-06T05:37:03.711Z" }, + { url = "https://files.pythonhosted.org/packages/fd/00/04ca1c3a7a124b6de4f8a9a17cc2fcad138b4608e7a3fc5877804b8715d7/frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", size = 43492, upload-time = "2025-10-06T05:37:04.915Z" }, + { url = "https://files.pythonhosted.org/packages/59/5e/c69f733a86a94ab10f68e496dc6b7e8bc078ebb415281d5698313e3af3a1/frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", size = 48034, upload-time = "2025-10-06T05:37:06.343Z" }, + { url = "https://files.pythonhosted.org/packages/16/6c/be9d79775d8abe79b05fa6d23da99ad6e7763a1d080fbae7290b286093fd/frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", size = 41749, upload-time = "2025-10-06T05:37:07.431Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" }, + { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" }, + { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" }, + { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" }, + { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" }, + { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" }, + { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" }, + { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" }, + { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" }, + { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" }, + { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" }, + { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" }, + { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" }, + { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" }, + { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" }, + { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" }, + { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" }, + { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + [[package]] name = "googleapis-common-protos" version = "1.75.1" @@ -1211,6 +1488,9 @@ lint = [ { name = "ruff" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] +load = [ + { name = "rampa", marker = "python_full_version >= '3.14'" }, +] otel = [ { name = "opentelemetry-exporter-otlp-proto-http" }, { name = "opentelemetry-sdk" }, @@ -1273,6 +1553,7 @@ lint = [ { name = "ruff", specifier = ">=0.16.1" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] +load = [{ name = "rampa", marker = "python_full_version >= '3.14'" }] otel = [ { name = "opentelemetry-exporter-otlp-proto-http" }, { name = "opentelemetry-sdk" }, @@ -1476,6 +1757,141 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192", size = 72226, upload-time = "2026-05-22T14:14:28.824Z" }, ] +[[package]] +name = "multidict" +version = "6.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/0b/19348d4c98980c4851d2f943f8ebafdece2ae7ef737adcfa5994ce8e5f10/multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5", size = 77176, upload-time = "2026-01-26T02:42:59.784Z" }, + { url = "https://files.pythonhosted.org/packages/ef/04/9de3f8077852e3d438215c81e9b691244532d2e05b4270e89ce67b7d103c/multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8", size = 44996, upload-time = "2026-01-26T02:43:01.674Z" }, + { url = "https://files.pythonhosted.org/packages/31/5c/08c7f7fe311f32e83f7621cd3f99d805f45519cd06fafb247628b861da7d/multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872", size = 44631, upload-time = "2026-01-26T02:43:03.169Z" }, + { url = "https://files.pythonhosted.org/packages/b7/7f/0e3b1390ae772f27501199996b94b52ceeb64fe6f9120a32c6c3f6b781be/multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991", size = 242561, upload-time = "2026-01-26T02:43:04.733Z" }, + { url = "https://files.pythonhosted.org/packages/dd/f4/8719f4f167586af317b69dd3e90f913416c91ca610cac79a45c53f590312/multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03", size = 242223, upload-time = "2026-01-26T02:43:06.695Z" }, + { url = "https://files.pythonhosted.org/packages/47/ab/7c36164cce64a6ad19c6d9a85377b7178ecf3b89f8fd589c73381a5eedfd/multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981", size = 222322, upload-time = "2026-01-26T02:43:08.472Z" }, + { url = "https://files.pythonhosted.org/packages/f5/79/a25add6fb38035b5337bc5734f296d9afc99163403bbcf56d4170f97eb62/multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6", size = 254005, upload-time = "2026-01-26T02:43:10.127Z" }, + { url = "https://files.pythonhosted.org/packages/4a/7b/64a87cf98e12f756fc8bd444b001232ffff2be37288f018ad0d3f0aae931/multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190", size = 251173, upload-time = "2026-01-26T02:43:11.731Z" }, + { url = "https://files.pythonhosted.org/packages/4b/ac/b605473de2bb404e742f2cc3583d12aedb2352a70e49ae8fce455b50c5aa/multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92", size = 243273, upload-time = "2026-01-26T02:43:13.063Z" }, + { url = "https://files.pythonhosted.org/packages/03/65/11492d6a0e259783720f3bc1d9ea55579a76f1407e31ed44045c99542004/multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee", size = 238956, upload-time = "2026-01-26T02:43:14.843Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a7/7ee591302af64e7c196fb63fe856c788993c1372df765102bd0448e7e165/multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2", size = 233477, upload-time = "2026-01-26T02:43:16.025Z" }, + { url = "https://files.pythonhosted.org/packages/9c/99/c109962d58756c35fd9992fed7f2355303846ea2ff054bb5f5e9d6b888de/multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568", size = 243615, upload-time = "2026-01-26T02:43:17.84Z" }, + { url = "https://files.pythonhosted.org/packages/d5/5f/1973e7c771c86e93dcfe1c9cc55a5481b610f6614acfc28c0d326fe6bfad/multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40", size = 249930, upload-time = "2026-01-26T02:43:19.06Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a5/f170fc2268c3243853580203378cd522446b2df632061e0a5409817854c7/multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962", size = 243807, upload-time = "2026-01-26T02:43:20.286Z" }, + { url = "https://files.pythonhosted.org/packages/de/01/73856fab6d125e5bc652c3986b90e8699a95e84b48d72f39ade6c0e74a8c/multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505", size = 239103, upload-time = "2026-01-26T02:43:21.508Z" }, + { url = "https://files.pythonhosted.org/packages/e7/46/f1220bd9944d8aa40d8ccff100eeeee19b505b857b6f603d6078cb5315b0/multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122", size = 41416, upload-time = "2026-01-26T02:43:22.703Z" }, + { url = "https://files.pythonhosted.org/packages/68/00/9b38e272a770303692fc406c36e1a4c740f401522d5787691eb38a8925a8/multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df", size = 46022, upload-time = "2026-01-26T02:43:23.77Z" }, + { url = "https://files.pythonhosted.org/packages/64/65/d8d42490c02ee07b6bbe00f7190d70bb4738b3cce7629aaf9f213ef730dd/multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db", size = 43238, upload-time = "2026-01-26T02:43:24.882Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f1/a90635c4f88fb913fbf4ce660b83b7445b7a02615bda034b2f8eb38fd597/multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d", size = 76626, upload-time = "2026-01-26T02:43:26.485Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/267e64eaf6fc637a15b35f5de31a566634a2740f97d8d094a69d34f524a4/multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e", size = 44706, upload-time = "2026-01-26T02:43:27.607Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a4/d45caf2b97b035c57267791ecfaafbd59c68212004b3842830954bb4b02e/multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855", size = 44356, upload-time = "2026-01-26T02:43:28.661Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d2/0a36c8473f0cbaeadd5db6c8b72d15bbceeec275807772bfcd059bef487d/multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3", size = 244355, upload-time = "2026-01-26T02:43:31.165Z" }, + { url = "https://files.pythonhosted.org/packages/5d/16/8c65be997fd7dd311b7d39c7b6e71a0cb449bad093761481eccbbe4b42a2/multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e", size = 246433, upload-time = "2026-01-26T02:43:32.581Z" }, + { url = "https://files.pythonhosted.org/packages/01/fb/4dbd7e848d2799c6a026ec88ad39cf2b8416aa167fcc903baa55ecaa045c/multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a", size = 225376, upload-time = "2026-01-26T02:43:34.417Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8a/4a3a6341eac3830f6053062f8fbc9a9e54407c80755b3f05bc427295c2d0/multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8", size = 257365, upload-time = "2026-01-26T02:43:35.741Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a2/dd575a69c1aa206e12d27d0770cdf9b92434b48a9ef0cd0d1afdecaa93c4/multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0", size = 254747, upload-time = "2026-01-26T02:43:36.976Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/21b27c560c13822ed93133f08aa6372c53a8e067f11fbed37b4adcdac922/multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144", size = 246293, upload-time = "2026-01-26T02:43:38.258Z" }, + { url = "https://files.pythonhosted.org/packages/5a/a4/23466059dc3854763423d0ad6c0f3683a379d97673b1b89ec33826e46728/multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49", size = 242962, upload-time = "2026-01-26T02:43:40.034Z" }, + { url = "https://files.pythonhosted.org/packages/1f/67/51dd754a3524d685958001e8fa20a0f5f90a6a856e0a9dcabff69be3dbb7/multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71", size = 237360, upload-time = "2026-01-26T02:43:41.752Z" }, + { url = "https://files.pythonhosted.org/packages/64/3f/036dfc8c174934d4b55d86ff4f978e558b0e585cef70cfc1ad01adc6bf18/multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3", size = 245940, upload-time = "2026-01-26T02:43:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/3d/20/6214d3c105928ebc353a1c644a6ef1408bc5794fcb4f170bb524a3c16311/multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c", size = 253502, upload-time = "2026-01-26T02:43:44.371Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e2/c653bc4ae1be70a0f836b82172d643fcf1dade042ba2676ab08ec08bff0f/multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0", size = 247065, upload-time = "2026-01-26T02:43:45.745Z" }, + { url = "https://files.pythonhosted.org/packages/c8/11/a854b4154cd3bd8b1fd375e8a8ca9d73be37610c361543d56f764109509b/multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa", size = 241870, upload-time = "2026-01-26T02:43:47.054Z" }, + { url = "https://files.pythonhosted.org/packages/13/bf/9676c0392309b5fdae322333d22a829715b570edb9baa8016a517b55b558/multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a", size = 41302, upload-time = "2026-01-26T02:43:48.753Z" }, + { url = "https://files.pythonhosted.org/packages/c9/68/f16a3a8ba6f7b6dc92a1f19669c0810bd2c43fc5a02da13b1cbf8e253845/multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b", size = 45981, upload-time = "2026-01-26T02:43:49.921Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/9dd5305253fa00cd3c7555dbef69d5bf4133debc53b87ab8d6a44d411665/multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6", size = 43159, upload-time = "2026-01-26T02:43:51.635Z" }, + { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893, upload-time = "2026-01-26T02:43:52.754Z" }, + { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456, upload-time = "2026-01-26T02:43:53.893Z" }, + { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872, upload-time = "2026-01-26T02:43:55.041Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018, upload-time = "2026-01-26T02:43:56.198Z" }, + { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883, upload-time = "2026-01-26T02:43:57.499Z" }, + { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413, upload-time = "2026-01-26T02:43:58.755Z" }, + { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404, upload-time = "2026-01-26T02:44:00.216Z" }, + { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456, upload-time = "2026-01-26T02:44:02.202Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, + { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955, upload-time = "2026-01-26T02:44:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254, upload-time = "2026-01-26T02:44:06.133Z" }, + { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059, upload-time = "2026-01-26T02:44:07.518Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588, upload-time = "2026-01-26T02:44:09.382Z" }, + { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642, upload-time = "2026-01-26T02:44:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887, upload-time = "2026-01-26T02:44:14.245Z" }, + { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053, upload-time = "2026-01-26T02:44:15.371Z" }, + { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307, upload-time = "2026-01-26T02:44:16.852Z" }, + { url = "https://files.pythonhosted.org/packages/f2/22/929c141d6c0dba87d3e1d38fbdf1ba8baba86b7776469f2bc2d3227a1e67/multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23", size = 76174, upload-time = "2026-01-26T02:44:18.509Z" }, + { url = "https://files.pythonhosted.org/packages/c7/75/bc704ae15fee974f8fccd871305e254754167dce5f9e42d88a2def741a1d/multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2", size = 45116, upload-time = "2026-01-26T02:44:19.745Z" }, + { url = "https://files.pythonhosted.org/packages/79/76/55cd7186f498ed080a18440c9013011eb548f77ae1b297206d030eb1180a/multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445", size = 43524, upload-time = "2026-01-26T02:44:21.571Z" }, + { url = "https://files.pythonhosted.org/packages/e9/3c/414842ef8d5a1628d68edee29ba0e5bcf235dbfb3ccd3ea303a7fe8c72ff/multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177", size = 249368, upload-time = "2026-01-26T02:44:22.803Z" }, + { url = "https://files.pythonhosted.org/packages/f6/32/befed7f74c458b4a525e60519fe8d87eef72bb1e99924fa2b0f9d97a221e/multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23", size = 256952, upload-time = "2026-01-26T02:44:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/03/d6/c878a44ba877f366630c860fdf74bfb203c33778f12b6ac274936853c451/multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060", size = 240317, upload-time = "2026-01-26T02:44:25.772Z" }, + { url = "https://files.pythonhosted.org/packages/68/49/57421b4d7ad2e9e60e25922b08ceb37e077b90444bde6ead629095327a6f/multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d", size = 267132, upload-time = "2026-01-26T02:44:27.648Z" }, + { url = "https://files.pythonhosted.org/packages/b7/fe/ec0edd52ddbcea2a2e89e174f0206444a61440b40f39704e64dc807a70bd/multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed", size = 268140, upload-time = "2026-01-26T02:44:29.588Z" }, + { url = "https://files.pythonhosted.org/packages/b0/73/6e1b01cbeb458807aa0831742232dbdd1fa92bfa33f52a3f176b4ff3dc11/multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429", size = 254277, upload-time = "2026-01-26T02:44:30.902Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b2/5fb8c124d7561a4974c342bc8c778b471ebbeb3cc17df696f034a7e9afe7/multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6", size = 252291, upload-time = "2026-01-26T02:44:32.31Z" }, + { url = "https://files.pythonhosted.org/packages/5a/96/51d4e4e06bcce92577fcd488e22600bd38e4fd59c20cb49434d054903bd2/multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9", size = 250156, upload-time = "2026-01-26T02:44:33.734Z" }, + { url = "https://files.pythonhosted.org/packages/db/6b/420e173eec5fba721a50e2a9f89eda89d9c98fded1124f8d5c675f7a0c0f/multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c", size = 249742, upload-time = "2026-01-26T02:44:35.222Z" }, + { url = "https://files.pythonhosted.org/packages/44/a3/ec5b5bd98f306bc2aa297b8c6f11a46714a56b1e6ef5ebda50a4f5d7c5fb/multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84", size = 262221, upload-time = "2026-01-26T02:44:36.604Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f7/e8c0d0da0cd1e28d10e624604e1a36bcc3353aaebdfdc3a43c72bc683a12/multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d", size = 258664, upload-time = "2026-01-26T02:44:38.008Z" }, + { url = "https://files.pythonhosted.org/packages/52/da/151a44e8016dd33feed44f730bd856a66257c1ee7aed4f44b649fb7edeb3/multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33", size = 249490, upload-time = "2026-01-26T02:44:39.386Z" }, + { url = "https://files.pythonhosted.org/packages/87/af/a3b86bf9630b732897f6fc3f4c4714b90aa4361983ccbdcd6c0339b21b0c/multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3", size = 41695, upload-time = "2026-01-26T02:44:41.318Z" }, + { url = "https://files.pythonhosted.org/packages/b2/35/e994121b0e90e46134673422dd564623f93304614f5d11886b1b3e06f503/multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5", size = 45884, upload-time = "2026-01-26T02:44:42.488Z" }, + { url = "https://files.pythonhosted.org/packages/ca/61/42d3e5dbf661242a69c97ea363f2d7b46c567da8eadef8890022be6e2ab0/multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df", size = 43122, upload-time = "2026-01-26T02:44:43.664Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b3/e6b21c6c4f314bb956016b0b3ef2162590a529b84cb831c257519e7fde44/multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1", size = 83175, upload-time = "2026-01-26T02:44:44.894Z" }, + { url = "https://files.pythonhosted.org/packages/fb/76/23ecd2abfe0957b234f6c960f4ade497f55f2c16aeb684d4ecdbf1c95791/multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963", size = 48460, upload-time = "2026-01-26T02:44:46.106Z" }, + { url = "https://files.pythonhosted.org/packages/c4/57/a0ed92b23f3a042c36bc4227b72b97eca803f5f1801c1ab77c8a212d455e/multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34", size = 46930, upload-time = "2026-01-26T02:44:47.278Z" }, + { url = "https://files.pythonhosted.org/packages/b5/66/02ec7ace29162e447f6382c495dc95826bf931d3818799bbef11e8f7df1a/multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65", size = 242582, upload-time = "2026-01-26T02:44:48.604Z" }, + { url = "https://files.pythonhosted.org/packages/58/18/64f5a795e7677670e872673aca234162514696274597b3708b2c0d276cce/multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292", size = 250031, upload-time = "2026-01-26T02:44:50.544Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ed/e192291dbbe51a8290c5686f482084d31bcd9d09af24f63358c3d42fd284/multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43", size = 228596, upload-time = "2026-01-26T02:44:51.951Z" }, + { url = "https://files.pythonhosted.org/packages/1e/7e/3562a15a60cf747397e7f2180b0a11dc0c38d9175a650e75fa1b4d325e15/multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca", size = 257492, upload-time = "2026-01-26T02:44:53.902Z" }, + { url = "https://files.pythonhosted.org/packages/24/02/7d0f9eae92b5249bb50ac1595b295f10e263dd0078ebb55115c31e0eaccd/multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd", size = 255899, upload-time = "2026-01-26T02:44:55.316Z" }, + { url = "https://files.pythonhosted.org/packages/00/e3/9b60ed9e23e64c73a5cde95269ef1330678e9c6e34dd4eb6b431b85b5a10/multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7", size = 247970, upload-time = "2026-01-26T02:44:56.783Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/538e58a63ed5cfb0bd4517e346b91da32fde409d839720f664e9a4ae4f9d/multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3", size = 245060, upload-time = "2026-01-26T02:44:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2f/d743a3045a97c895d401e9bd29aaa09b94f5cbdf1bd561609e5a6c431c70/multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4", size = 235888, upload-time = "2026-01-26T02:44:59.57Z" }, + { url = "https://files.pythonhosted.org/packages/38/83/5a325cac191ab28b63c52f14f1131f3b0a55ba3b9aa65a6d0bf2a9b921a0/multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8", size = 243554, upload-time = "2026-01-26T02:45:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/9d2327086bd15da2725ef6aae624208e2ef828ed99892b17f60c344e57ed/multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c", size = 252341, upload-time = "2026-01-26T02:45:02.484Z" }, + { url = "https://files.pythonhosted.org/packages/e8/2c/2a1aa0280cf579d0f6eed8ee5211c4f1730bd7e06c636ba2ee6aafda302e/multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52", size = 246391, upload-time = "2026-01-26T02:45:03.862Z" }, + { url = "https://files.pythonhosted.org/packages/e5/03/7ca022ffc36c5a3f6e03b179a5ceb829be9da5783e6fe395f347c0794680/multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108", size = 243422, upload-time = "2026-01-26T02:45:05.296Z" }, + { url = "https://files.pythonhosted.org/packages/dc/1d/b31650eab6c5778aceed46ba735bd97f7c7d2f54b319fa916c0f96e7805b/multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32", size = 47770, upload-time = "2026-01-26T02:45:06.754Z" }, + { url = "https://files.pythonhosted.org/packages/ac/5b/2d2d1d522e51285bd61b1e20df8f47ae1a9d80839db0b24ea783b3832832/multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8", size = 53109, upload-time = "2026-01-26T02:45:08.044Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a3/cc409ba012c83ca024a308516703cf339bdc4b696195644a7215a5164a24/multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118", size = 45573, upload-time = "2026-01-26T02:45:09.349Z" }, + { url = "https://files.pythonhosted.org/packages/91/cc/db74228a8be41884a567e88a62fd589a913708fcf180d029898c17a9a371/multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee", size = 75190, upload-time = "2026-01-26T02:45:10.651Z" }, + { url = "https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2", size = 44486, upload-time = "2026-01-26T02:45:11.938Z" }, + { url = "https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1", size = 43219, upload-time = "2026-01-26T02:45:14.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/bb/2c0c2287963f4259c85e8bcbba9182ced8d7fca65c780c38e99e61629d11/multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d", size = 245132, upload-time = "2026-01-26T02:45:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31", size = 252420, upload-time = "2026-01-26T02:45:17.293Z" }, + { url = "https://files.pythonhosted.org/packages/8b/13/78f7275e73fa17b24c9a51b0bd9d73ba64bb32d0ed51b02a746eb876abe7/multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048", size = 233510, upload-time = "2026-01-26T02:45:19.356Z" }, + { url = "https://files.pythonhosted.org/packages/4b/25/8167187f62ae3cbd52da7893f58cb036b47ea3fb67138787c76800158982/multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362", size = 264094, upload-time = "2026-01-26T02:45:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e7/69a3a83b7b030cf283fb06ce074a05a02322359783424d7edf0f15fe5022/multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37", size = 260786, upload-time = "2026-01-26T02:45:22.818Z" }, + { url = "https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709", size = 248483, upload-time = "2026-01-26T02:45:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/d5a99e3acbca0e29c5d9cba8f92ceb15dce78bab963b308ae692981e3a5d/multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0", size = 248403, upload-time = "2026-01-26T02:45:25.982Z" }, + { url = "https://files.pythonhosted.org/packages/35/48/e58cd31f6c7d5102f2a4bf89f96b9cf7e00b6c6f3d04ecc44417c00a5a3c/multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb", size = 240315, upload-time = "2026-01-26T02:45:27.487Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/1cd210229559cb90b6786c30676bb0c58249ff42f942765f88793b41fdce/multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd", size = 245528, upload-time = "2026-01-26T02:45:28.991Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/6e1107d226278c876c783056b7db43d800bb64c6131cec9c8dfb6903698e/multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601", size = 258784, upload-time = "2026-01-26T02:45:30.503Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c1/11f664f14d525e4a1b5327a82d4de61a1db604ab34c6603bb3c2cc63ad34/multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1", size = 251980, upload-time = "2026-01-26T02:45:32.603Z" }, + { url = "https://files.pythonhosted.org/packages/e1/9f/75a9ac888121d0c5bbd4ecf4eead45668b1766f6baabfb3b7f66a410e231/multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b", size = 243602, upload-time = "2026-01-26T02:45:34.043Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e7/50bf7b004cc8525d80dbbbedfdc7aed3e4c323810890be4413e589074032/multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d", size = 40930, upload-time = "2026-01-26T02:45:36.278Z" }, + { url = "https://files.pythonhosted.org/packages/e0/bf/52f25716bbe93745595800f36fb17b73711f14da59ed0bb2eba141bc9f0f/multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f", size = 45074, upload-time = "2026-01-26T02:45:37.546Z" }, + { url = "https://files.pythonhosted.org/packages/97/ab/22803b03285fa3a525f48217963da3a65ae40f6a1b6f6cf2768879e208f9/multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5", size = 42471, upload-time = "2026-01-26T02:45:38.889Z" }, + { url = "https://files.pythonhosted.org/packages/e0/6d/f9293baa6146ba9507e360ea0292b6422b016907c393e2f63fc40ab7b7b5/multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581", size = 82401, upload-time = "2026-01-26T02:45:40.254Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/53b5494738d83558d87c3c71a486504d8373421c3e0dbb6d0db48ad42ee0/multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a", size = 48143, upload-time = "2026-01-26T02:45:41.635Z" }, + { url = "https://files.pythonhosted.org/packages/37/e8/5284c53310dcdc99ce5d66563f6e5773531a9b9fe9ec7a615e9bc306b05f/multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c", size = 46507, upload-time = "2026-01-26T02:45:42.99Z" }, + { url = "https://files.pythonhosted.org/packages/e4/fc/6800d0e5b3875568b4083ecf5f310dcf91d86d52573160834fb4bfcf5e4f/multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262", size = 239358, upload-time = "2026-01-26T02:45:44.376Z" }, + { url = "https://files.pythonhosted.org/packages/41/75/4ad0973179361cdf3a113905e6e088173198349131be2b390f9fa4da5fc6/multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59", size = 246884, upload-time = "2026-01-26T02:45:47.167Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9c/095bb28b5da139bd41fb9a5d5caff412584f377914bd8787c2aa98717130/multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889", size = 225878, upload-time = "2026-01-26T02:45:48.698Z" }, + { url = "https://files.pythonhosted.org/packages/07/d0/c0a72000243756e8f5a277b6b514fa005f2c73d481b7d9e47cd4568aa2e4/multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4", size = 253542, upload-time = "2026-01-26T02:45:50.164Z" }, + { url = "https://files.pythonhosted.org/packages/c0/6b/f69da15289e384ecf2a68837ec8b5ad8c33e973aa18b266f50fe55f24b8c/multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d", size = 252403, upload-time = "2026-01-26T02:45:51.779Z" }, + { url = "https://files.pythonhosted.org/packages/a2/76/b9669547afa5a1a25cd93eaca91c0da1c095b06b6d2d8ec25b713588d3a1/multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609", size = 244889, upload-time = "2026-01-26T02:45:53.27Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a9/a50d2669e506dad33cfc45b5d574a205587b7b8a5f426f2fbb2e90882588/multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489", size = 241982, upload-time = "2026-01-26T02:45:54.919Z" }, + { url = "https://files.pythonhosted.org/packages/c5/bb/1609558ad8b456b4827d3c5a5b775c93b87878fd3117ed3db3423dfbce1b/multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c", size = 232415, upload-time = "2026-01-26T02:45:56.981Z" }, + { url = "https://files.pythonhosted.org/packages/d8/59/6f61039d2aa9261871e03ab9dc058a550d240f25859b05b67fd70f80d4b3/multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e", size = 240337, upload-time = "2026-01-26T02:45:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/a1/29/fdc6a43c203890dc2ae9249971ecd0c41deaedfe00d25cb6564b2edd99eb/multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c", size = 248788, upload-time = "2026-01-26T02:46:00.862Z" }, + { url = "https://files.pythonhosted.org/packages/a9/14/a153a06101323e4cf086ecee3faadba52ff71633d471f9685c42e3736163/multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9", size = 242842, upload-time = "2026-01-26T02:46:02.824Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/604ae839e64a4a6efc80db94465348d3b328ee955e37acb24badbcd24d83/multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2", size = 240237, upload-time = "2026-01-26T02:46:05.898Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/c3a5187bf66f6fb546ff4ab8fb5a077cbdd832d7b1908d4365c7f74a1917/multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7", size = 48008, upload-time = "2026-01-26T02:46:07.468Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f7/addf1087b860ac60e6f382240f64fb99f8bfb532bb06f7c542b83c29ca61/multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5", size = 53542, upload-time = "2026-01-26T02:46:08.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/81/4629d0aa32302ef7b2ec65c75a728cc5ff4fa410c50096174c1632e70b3e/multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2", size = 44719, upload-time = "2026-01-26T02:46:11.146Z" }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, +] + [[package]] name = "mypy" version = "2.3.0" @@ -1727,6 +2143,134 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "propcache" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/56/030b7b4719d53085722893e0009dffb9236aa10bca1b12121bdc5626ef16/propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b", size = 93417, upload-time = "2026-05-08T20:59:15.597Z" }, + { url = "https://files.pythonhosted.org/packages/1a/55/1140a8e067b8ec093a18a4ae7bb0045d9db65da38a08618ddc5e2f1994aa/propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c", size = 53847, upload-time = "2026-05-08T20:59:17.096Z" }, + { url = "https://files.pythonhosted.org/packages/20/42/0e7443c90310498561addf346e7d57fe3c6ba1914e1ba938b5464c7bbfd2/propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb", size = 53512, upload-time = "2026-05-08T20:59:18.64Z" }, + { url = "https://files.pythonhosted.org/packages/b7/db/cf51a71bab2009517d1a7f0ee07657e3bd446c4d69f67e6966cf17bcf956/propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e", size = 58068, upload-time = "2026-05-08T20:59:20.683Z" }, + { url = "https://files.pythonhosted.org/packages/b7/43/39b6bdee9699fa1e1641c519feeb64a67e2a9f93bb465c70776b37a7333f/propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e", size = 61020, upload-time = "2026-05-08T20:59:22.112Z" }, + { url = "https://files.pythonhosted.org/packages/26/0b/843726fbb0a29a8c5684fdb25971823638399f31e52e9d1f06a02dc9aa6b/propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b", size = 62732, upload-time = "2026-05-08T20:59:23.805Z" }, + { url = "https://files.pythonhosted.org/packages/39/6e/899fed76dc1942b8a64193a4f059d7f1a2c7ef65085e8a9366ed8ec0d199/propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d", size = 60140, upload-time = "2026-05-08T20:59:25.389Z" }, + { url = "https://files.pythonhosted.org/packages/ab/09/3da4be9b5b879219ad234aa535b3dd4a080ed1ad48d3a73ca07a9e798f22/propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d", size = 60400, upload-time = "2026-05-08T20:59:27.238Z" }, + { url = "https://files.pythonhosted.org/packages/60/2f/09b72b874a9aa0044faf52a69807a6ed618e267ceaa9ec4a63195fa5b504/propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0", size = 58155, upload-time = "2026-05-08T20:59:28.48Z" }, + { url = "https://files.pythonhosted.org/packages/8a/37/97489848c54c95578045473954f10956d619ce6a09e7ac137b71cdcb698b/propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b", size = 57037, upload-time = "2026-05-08T20:59:30.146Z" }, + { url = "https://files.pythonhosted.org/packages/22/db/6c695285ccfc49012743ee9c98212b8c5dd0aed7b63cfd816d4a0f7a1601/propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf", size = 61103, upload-time = "2026-05-08T20:59:31.626Z" }, + { url = "https://files.pythonhosted.org/packages/98/a9/1e500401ca593b0bdb6bf75a70bc2d723835fd53360edff6af70692c7546/propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf", size = 60394, upload-time = "2026-05-08T20:59:32.829Z" }, + { url = "https://files.pythonhosted.org/packages/1f/87/f638b6e375eae0f30a1a2325d8b34fd85fdc785bb9960cf805f3bf1ec69a/propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e", size = 63084, upload-time = "2026-05-08T20:59:35.964Z" }, + { url = "https://files.pythonhosted.org/packages/f6/18/884573f5d97b6d9eba68de759a82c901b7e39d7904d30f7b8d58d42d2a12/propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274", size = 60999, upload-time = "2026-05-08T20:59:38.481Z" }, + { url = "https://files.pythonhosted.org/packages/8f/1a/c3915eb059ceec9e758a56e4cfd955292bc0f201be2176a46b76d94b303a/propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe", size = 39036, upload-time = "2026-05-08T20:59:40.323Z" }, + { url = "https://files.pythonhosted.org/packages/5b/02/1dfd5607501a602d19c1c449d2d193b7d1c611f9246b4059026a1189a80e/propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d", size = 42190, upload-time = "2026-05-08T20:59:42.232Z" }, + { url = "https://files.pythonhosted.org/packages/57/93/f71588ad08b3e6f4b555b5ef215808a3c02b042d0151ad82fa6f15be677a/propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5", size = 38545, upload-time = "2026-05-08T20:59:44.087Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f1/8a8cc1c2c7e7934ab77e0163414f736fadbc0f5e8dd9673b952355ac175b/propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78", size = 90744, upload-time = "2026-05-08T20:59:45.799Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f4/651b1225e976bd1a2ba5cfba0c29d096581c2636b437e3a9a7ab6276270a/propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959", size = 52033, upload-time = "2026-05-08T20:59:47.408Z" }, + { url = "https://files.pythonhosted.org/packages/15/a8/8ede85d6aa1f79fc7dc2f8fd2c8d65920b8272c3892903c8a1affde48cfb/propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7", size = 52754, upload-time = "2026-05-08T20:59:49.202Z" }, + { url = "https://files.pythonhosted.org/packages/7d/fe/b3551b41bbc2f5b5bb088fc6920567cd43101253e68fbaa261339eb96fe1/propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511", size = 57573, upload-time = "2026-05-08T20:59:50.778Z" }, + { url = "https://files.pythonhosted.org/packages/83/27/ab851ebd1b7172e3e161f5f8d39e315d54a91bea246f01f4d872d3376aef/propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660", size = 60645, upload-time = "2026-05-08T20:59:52.227Z" }, + { url = "https://files.pythonhosted.org/packages/95/7d/466b3d18022e9897cbda9c735c493c5bd747d7a4c6f5ea1480b4cec434b6/propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66", size = 61563, upload-time = "2026-05-08T20:59:53.866Z" }, + { url = "https://files.pythonhosted.org/packages/27/1b/16ab7f2cf2041da2f60d156ba64c2484eadf9168075b4ff43c3ef60045af/propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b", size = 58888, upload-time = "2026-05-08T20:59:55.457Z" }, + { url = "https://files.pythonhosted.org/packages/0a/67/bb777ffd907633563bf35fd859c4ce97b0512c32f4633cf5d1eb7c33512b/propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67", size = 59253, upload-time = "2026-05-08T20:59:57.075Z" }, + { url = "https://files.pythonhosted.org/packages/b9/42/64f8d90b73fd9cdc1499b48057ff6d9cd2a98a25734c9bb62ecf07e87061/propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f", size = 57558, upload-time = "2026-05-08T20:59:58.602Z" }, + { url = "https://files.pythonhosted.org/packages/eb/02/dba5bc03c9041f2092ea55a449caf5dfe68352c6654511b29ba0654ddb69/propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c", size = 55007, upload-time = "2026-05-08T20:59:59.837Z" }, + { url = "https://files.pythonhosted.org/packages/14/c0/43f649c7aa2a77a3b100d84e9dea3a483120ecb608bfe36ce49eaff517fe/propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0", size = 60355, upload-time = "2026-05-08T21:00:01.144Z" }, + { url = "https://files.pythonhosted.org/packages/83/c0/435dafd27f1cb4a495381dae60e25883ccfe4020bb72818e8184c1678092/propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6", size = 59057, upload-time = "2026-05-08T21:00:02.401Z" }, + { url = "https://files.pythonhosted.org/packages/53/ae/6e292df9135d659944e96cb3389258e4a663e5b2b5f6c217ef0ddc8d2f73/propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27", size = 61938, upload-time = "2026-05-08T21:00:03.638Z" }, + { url = "https://files.pythonhosted.org/packages/0b/42/314ebc50d8159055411fd6b0bda322ff510e4b1f7d2e4927940ad0f6af20/propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f", size = 59731, upload-time = "2026-05-08T21:00:04.881Z" }, + { url = "https://files.pythonhosted.org/packages/b8/9b/2da6dee38871c3c8772fabc2758325a5c9077d6d18c597737dc04dd884cd/propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0", size = 38966, upload-time = "2026-05-08T21:00:06.511Z" }, + { url = "https://files.pythonhosted.org/packages/42/4e/f17363fb58c0afe05b067361cb6d86ed2d29de6506779a27547c4d183075/propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82", size = 42135, upload-time = "2026-05-08T21:00:08.088Z" }, + { url = "https://files.pythonhosted.org/packages/c6/eb/6af6685077d22e8b33358d3c548e3282706a0b3cd85044ffba4e5dd08e3b/propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab", size = 38381, upload-time = "2026-05-08T21:00:09.692Z" }, + { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, + { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, + { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, + { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, + { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, + { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, + { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, + { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, + { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, + { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, + { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, + { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, + { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, + { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, + { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, + { url = "https://files.pythonhosted.org/packages/c5/09/f049e45385503fe67db75a6b6186a7b9f0c3930366dc960522c312a825b1/propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a", size = 94457, upload-time = "2026-05-08T21:00:36.355Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/83d1d05655baf63113731bd5a1008435e14f8d1e5a06cbe4ec5b23ad7a31/propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117", size = 53835, upload-time = "2026-05-08T21:00:38.072Z" }, + { url = "https://files.pythonhosted.org/packages/a9/12/a6ba6482bb5ea3260c000c9b20881c95fa11c6b30173715668259f844ed7/propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098", size = 54545, upload-time = "2026-05-08T21:00:39.319Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/7fa086f5764c59ec8a8e157cd93aa8497acc00aba9dcdec56bfffb32602d/propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4", size = 59886, upload-time = "2026-05-08T21:00:40.621Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e4/5d7663dc8235956c8f5281698a3af1d351d8820341ddd890f59d9a9127f2/propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e", size = 63261, upload-time = "2026-05-08T21:00:41.775Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/15a03adee24d6350da4292caeac44c34c033d2afe5e87eb370f38854560f/propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7", size = 64184, upload-time = "2026-05-08T21:00:43.018Z" }, + { url = "https://files.pythonhosted.org/packages/8b/c6/979176efdaa3d239e36d503d5af63a0a773b36662ed8f52e5b6a6d9fd40e/propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d", size = 61534, upload-time = "2026-05-08T21:00:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/c8/22/63e8cd1bae4c2d2be6493b6b7d10566ddafad88137cfbc99964a1119853c/propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a", size = 61500, upload-time = "2026-05-08T21:00:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/60/5a/28e5d9acbac1cc9ccb67045e8c1b943aa8d79fdf39c93bd73cacd68008ea/propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2", size = 59994, upload-time = "2026-05-08T21:00:47.093Z" }, + { url = "https://files.pythonhosted.org/packages/f3/40/db650677f554a95b9c01a7c9d93d629e93a15562f5deb4573c9ee136fed2/propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa", size = 56884, upload-time = "2026-05-08T21:00:48.376Z" }, + { url = "https://files.pythonhosted.org/packages/80/45/70b39b89516ff8b96bf732fa6fded8cef20f293cb1508690101c3c07ec51/propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853", size = 63464, upload-time = "2026-05-08T21:00:49.954Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e2/fa59d3a89eac5534293124af4f1d0d0ada091ce4a0ab4610ce03fd2bdd8d/propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a", size = 61588, upload-time = "2026-05-08T21:00:51.281Z" }, + { url = "https://files.pythonhosted.org/packages/0b/97/efb547a55c4bc7381cfb202d6a2239ac621045277bc1ea5dfd3a7f0516c0/propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704", size = 64667, upload-time = "2026-05-08T21:00:52.602Z" }, + { url = "https://files.pythonhosted.org/packages/92/56/f5c7d9b4b7595d5127da38974d791b2153f3d1eae6c674af3583ace92ad3/propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4", size = 62463, upload-time = "2026-05-08T21:00:54.303Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/484a3a65fc9f9f60c41dcd17b428bace5389544e2c680994534a20755066/propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d", size = 38621, upload-time = "2026-05-08T21:00:55.808Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fd/3f0f10dba4dabad3bf53102be007abf55481067952bde0fdddff439e7c61/propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757", size = 41649, upload-time = "2026-05-08T21:00:57.061Z" }, + { url = "https://files.pythonhosted.org/packages/90/ec/6ce619cc32bb500a482f811f9cd509368b4e58e638d13f2c68f370d6b475/propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f", size = 37636, upload-time = "2026-05-08T21:00:58.646Z" }, + { url = "https://files.pythonhosted.org/packages/1b/82/c1d268bbbf2ef981c5bf0fbbe746db617c66e3bcefe431a1aa8943fbe23a/propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d", size = 98872, upload-time = "2026-05-08T21:00:59.889Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/52c871e73e864e6b34c0e2d58ac1ec5ccd149497ddc7ad2137ae98323a35/propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa", size = 56257, upload-time = "2026-05-08T21:01:01.195Z" }, + { url = "https://files.pythonhosted.org/packages/67/f0/9b90ca2a210b3d09bcfcd96ecd0f55545c091535abce2a45de2775cfd357/propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94", size = 56696, upload-time = "2026-05-08T21:01:02.941Z" }, + { url = "https://files.pythonhosted.org/packages/9d/0e/6e9d4ba07c8e56e21ddec1e75f12148142b21ca83a51871babce095334f4/propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164", size = 62378, upload-time = "2026-05-08T21:01:04.475Z" }, + { url = "https://files.pythonhosted.org/packages/65/19/c10badaa463dde8a27ce884f8ee2ec37e6035b7c9f5ff0c8f74f06f08dac/propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f", size = 65283, upload-time = "2026-05-08T21:01:05.959Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b6/93bea99ca80e19cef6512a8580e5b7857bbe09422d9daa7fd4ef5723306c/propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c", size = 66616, upload-time = "2026-05-08T21:01:07.228Z" }, + { url = "https://files.pythonhosted.org/packages/83/e4/5c7462e50625f051f37fb38b8224f7639f667184bbd34424ec83819bb1b7/propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc", size = 63773, upload-time = "2026-05-08T21:01:08.514Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b6/99238894047b13c823be25027e736626cd414a52a5e30d2c3347c2733529/propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f", size = 63664, upload-time = "2026-05-08T21:01:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/85/1e/a3a1a63116a2b8edb415a8bb9a6f0c34bd03830b1e18e8ce2904e1dc1cf4/propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb", size = 62643, upload-time = "2026-05-08T21:01:11.132Z" }, + { url = "https://files.pythonhosted.org/packages/e4/03/893cf147de2fc6543c5eaa07ad833170e7e2a2385725bbebe8c0503723bb/propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751", size = 59595, upload-time = "2026-05-08T21:01:12.387Z" }, + { url = "https://files.pythonhosted.org/packages/86/3b/04c1a2e12c57766568ba75ba72b3bf2042818d4c1425fab6fc07155c7cff/propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836", size = 65711, upload-time = "2026-05-08T21:01:13.676Z" }, + { url = "https://files.pythonhosted.org/packages/1c/34/80f8d0099f8d6bacc4de1624c85672681c8cd1149ca2da0e38fd120b817f/propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f", size = 64247, upload-time = "2026-05-08T21:01:14.936Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1a/8b08f3a5f1037e9e370c55883ceeeee0f6dd0416fb2d2d67b8bfc91f2a79/propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55", size = 67102, upload-time = "2026-05-08T21:01:16.281Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/8bdb7bb7756d76e005490649d10e4a8369e610c74d619f71e1aedf889e9c/propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568", size = 64964, upload-time = "2026-05-08T21:01:17.57Z" }, + { url = "https://files.pythonhosted.org/packages/0a/aa/50fb0b5d3968b61a510926ff8b8465f1d6e976b3ab74496d7a4b9fc42515/propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191", size = 42546, upload-time = "2026-05-08T21:01:18.946Z" }, + { url = "https://files.pythonhosted.org/packages/ae/4c/0ddbae64321bd4a95bcbfc19307238016b5b1fee645c84626c8d539e5b74/propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7", size = 46330, upload-time = "2026-05-08T21:01:20.162Z" }, + { url = "https://files.pythonhosted.org/packages/00/d9/9cddc8efb78d8af264c5ec9f6d10b62f57c515feda8d321595f56010fb23/propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96", size = 40521, upload-time = "2026-05-08T21:01:21.399Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662, upload-time = "2026-05-08T21:01:22.683Z" }, + { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928, upload-time = "2026-05-08T21:01:23.986Z" }, + { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650, upload-time = "2026-05-08T21:01:25.305Z" }, + { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912, upload-time = "2026-05-08T21:01:26.545Z" }, + { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300, upload-time = "2026-05-08T21:01:27.937Z" }, + { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208, upload-time = "2026-05-08T21:01:29.484Z" }, + { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633, upload-time = "2026-05-08T21:01:31.068Z" }, + { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724, upload-time = "2026-05-08T21:01:32.374Z" }, + { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069, upload-time = "2026-05-08T21:01:33.67Z" }, + { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099, upload-time = "2026-05-08T21:01:34.915Z" }, + { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391, upload-time = "2026-05-08T21:01:36.231Z" }, + { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626, upload-time = "2026-05-08T21:01:37.545Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781, upload-time = "2026-05-08T21:01:38.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570, upload-time = "2026-05-08T21:01:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436, upload-time = "2026-05-08T21:01:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373, upload-time = "2026-05-08T21:01:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554, upload-time = "2026-05-08T21:01:44.336Z" }, + { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395, upload-time = "2026-05-08T21:01:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653, upload-time = "2026-05-08T21:01:47.307Z" }, + { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914, upload-time = "2026-05-08T21:01:48.573Z" }, + { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567, upload-time = "2026-05-08T21:01:49.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542, upload-time = "2026-05-08T21:01:51.204Z" }, + { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845, upload-time = "2026-05-08T21:01:52.539Z" }, + { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985, upload-time = "2026-05-08T21:01:53.847Z" }, + { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999, upload-time = "2026-05-08T21:01:55.179Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779, upload-time = "2026-05-08T21:01:57.489Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796, upload-time = "2026-05-08T21:01:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023, upload-time = "2026-05-08T21:02:00.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448, upload-time = "2026-05-08T21:02:01.888Z" }, + { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329, upload-time = "2026-05-08T21:02:03.484Z" }, + { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172, upload-time = "2026-05-08T21:02:04.745Z" }, + { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813, upload-time = "2026-05-08T21:02:06.025Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764, upload-time = "2026-05-08T21:02:07.353Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140, upload-time = "2026-05-08T21:02:09.065Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, +] + [[package]] name = "protobuf" version = "7.35.1" @@ -2199,6 +2743,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "rampa" +version = "0.0.1a1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/62/ac3178d20316d13a9385ca0bde81b134a47cd10ac3d3a9645e41b50dd340/rampa-0.0.1a1.tar.gz", hash = "sha256:f63b8fb40ceef4b92211b96ecaef0495f216f11c620f9b30225215b809f1d587", size = 71636, upload-time = "2026-05-27T04:25:49.448Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/13/43da9e3bef942e3d4973a984175ce9a67429b942638e4497f1b5bc2756f4/rampa-0.0.1a1-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:a812cb0086f44aa85d9d1fd04aa76dcd61f63a04253f2b95ddc8f9996f4f148d", size = 371864, upload-time = "2026-05-27T04:25:47.963Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -3439,6 +3996,122 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/4d/2d0d67834092e354d2b0498f014a41249a89556bc406cf86f3e1557bb463/websockets-16.1.1-py3-none-any.whl", hash = "sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3", size = 173814, upload-time = "2026-07-17T22:51:04.184Z" }, ] +[[package]] +name = "yarl" +version = "1.24.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/33/ebe9e3d1f86c7a0b51094c0a146392045ca1631d2664889539dec8088a33/yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f", size = 228679, upload-time = "2026-07-20T02:07:45.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/ac/cacdda1f0a90441297210bc34cf7e4ac1b7318c8030ebd83bdf6fe82f1db/yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750", size = 135466, upload-time = "2026-07-20T02:04:21.695Z" }, + { url = "https://files.pythonhosted.org/packages/c0/a5/1b2ceace0230e40c52ab1b263148059a43a6303219b996affc68f8381836/yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2", size = 97291, upload-time = "2026-07-20T02:04:24.045Z" }, + { url = "https://files.pythonhosted.org/packages/59/1d/340d1a0db7bbce1f291afc044255ebf4ebbce2b25ab1b3f7d3d069080f5d/yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871", size = 97154, upload-time = "2026-07-20T02:04:25.761Z" }, + { url = "https://files.pythonhosted.org/packages/05/41/25596a33c2fb5098dca8dc3773b04221db64ded0b7f8f09885647d864610/yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0", size = 109196, upload-time = "2026-07-20T02:04:27.543Z" }, + { url = "https://files.pythonhosted.org/packages/f2/df/dd9f2fb8a5c6054fbefd1538d2b9b1127e612d2ee64b307a070173b57afd/yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e", size = 102556, upload-time = "2026-07-20T02:04:29.16Z" }, + { url = "https://files.pythonhosted.org/packages/cb/57/4754b9d2c8945880290ecba0864e8b0441e117bba70534fe819e3645e174/yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2", size = 117965, upload-time = "2026-07-20T02:04:30.845Z" }, + { url = "https://files.pythonhosted.org/packages/74/b5/6a9ece27d2043c3386f902dd078ab35d29ef5126b3206ebffb673283a7cb/yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621", size = 116266, upload-time = "2026-07-20T02:04:32.573Z" }, + { url = "https://files.pythonhosted.org/packages/9e/bc/a6653249f6ee59ec85dcfec008d9cbc16586dad613963bb17a91b2b993a5/yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba", size = 110758, upload-time = "2026-07-20T02:04:34.235Z" }, + { url = "https://files.pythonhosted.org/packages/65/7e/c5a12fb8208df7b981bc82256e7831ce428eeaf893f7bbe6179c57bb9252/yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950", size = 110120, upload-time = "2026-07-20T02:04:35.85Z" }, + { url = "https://files.pythonhosted.org/packages/04/6c/1b659b964626694667b3ec01bf4bcff564b73ae7c48ea1fbfe588b78b461/yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00", size = 108834, upload-time = "2026-07-20T02:04:37.67Z" }, + { url = "https://files.pythonhosted.org/packages/74/a6/bf48f55c2104e40c15b7b13fad0a5756a11552a55f01c90bc90a66ab81c3/yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed", size = 103442, upload-time = "2026-07-20T02:04:39.576Z" }, + { url = "https://files.pythonhosted.org/packages/37/ac/84b273ac133ecdce598fc1f4140a08a1bf2044048bff8106371d207d105f/yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440", size = 117413, upload-time = "2026-07-20T02:04:41.549Z" }, + { url = "https://files.pythonhosted.org/packages/a4/55/9307e03977d3b290dfa42e5d2bae7b6140808fd1786fbe70cd9d3bee53c5/yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1", size = 109498, upload-time = "2026-07-20T02:04:43.468Z" }, + { url = "https://files.pythonhosted.org/packages/fc/be/791a6f314cb4c989c19f8e3a10271f1e469c077143915e52474d80f26b4b/yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6", size = 116062, upload-time = "2026-07-20T02:04:45.319Z" }, + { url = "https://files.pythonhosted.org/packages/19/1a/ddd3807b86055010e2f99aa89b3c640effdb65696766c20597f696f48a1c/yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d", size = 110941, upload-time = "2026-07-20T02:04:47.054Z" }, + { url = "https://files.pythonhosted.org/packages/6d/03/f34271bba042d2187508bf62aea20a14129efb5a1acfc6a2efe7544630b4/yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224", size = 97534, upload-time = "2026-07-20T02:04:48.774Z" }, + { url = "https://files.pythonhosted.org/packages/e4/02/ecc8dc31b9f355731e700f8402b8075d2ea1737dbc4baf4abf0f0fc64288/yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13", size = 93603, upload-time = "2026-07-20T02:04:50.686Z" }, + { url = "https://files.pythonhosted.org/packages/fe/db/3cb5df059756a45761cc3dee8fd25ec82b83a6585ea3542b969fda850f99/yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3", size = 135043, upload-time = "2026-07-20T02:04:52.39Z" }, + { url = "https://files.pythonhosted.org/packages/44/f8/767d6bd5a03db63bc467df2fb56d6fafeae9667d74aea92cd6af399f828b/yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a", size = 96942, upload-time = "2026-07-20T02:04:54.26Z" }, + { url = "https://files.pythonhosted.org/packages/ce/97/10b939c44d7b28d1dbc389cfc7012306d1ea8dba01eaef44b39fffaee52a/yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840", size = 97046, upload-time = "2026-07-20T02:04:56.638Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7a/b410dbe39b6255c55fb2a2bcee96eb844d0789235ddc381a889a90dc72d6/yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966", size = 110512, upload-time = "2026-07-20T02:04:58.955Z" }, + { url = "https://files.pythonhosted.org/packages/83/c7/da591971f78a5617e1f21f5699858ebccd836fe181a6493788ffc91ba69b/yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723", size = 102454, upload-time = "2026-07-20T02:05:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8e/73b0ed4de47289a78a96045d76d1cfe5e41848bf0da59ce25b2ec87ee05d/yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb", size = 117617, upload-time = "2026-07-20T02:05:02.325Z" }, + { url = "https://files.pythonhosted.org/packages/cf/14/b744747bc4f57a8d55bd744df463457524583e1e9f7538b5ace0346ab92e/yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780", size = 116135, upload-time = "2026-07-20T02:05:04.05Z" }, + { url = "https://files.pythonhosted.org/packages/66/ca/95aa4d0e5b7ea4f20e4d577c42d001ed9df207569fdb063cc5ed4ebb496b/yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e", size = 111935, upload-time = "2026-07-20T02:05:05.738Z" }, + { url = "https://files.pythonhosted.org/packages/72/0d/d2ad8d6b147832d177a4e720ba1962fe686eb0913b74503b3eca094b8bba/yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2", size = 110010, upload-time = "2026-07-20T02:05:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/50/18/eb335e4120903903f4865041355ae46256a2406eb2865bc24827f4f27b61/yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58", size = 110058, upload-time = "2026-07-20T02:05:09.246Z" }, + { url = "https://files.pythonhosted.org/packages/44/70/97353add32c62ad6f206d948ac5a5ee84398225e534dc6ed6433d1b335b6/yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61", size = 103308, upload-time = "2026-07-20T02:05:11.31Z" }, + { url = "https://files.pythonhosted.org/packages/68/39/5e7398d4b6f6b3c9062823ebc60802df5b272e3fe9e788f9734c6ee46c85/yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6", size = 116898, upload-time = "2026-07-20T02:05:13.099Z" }, + { url = "https://files.pythonhosted.org/packages/e4/c9/09e52f2239e8b96357eccca05915382e4ba5405ebfb623b6036040d99654/yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f", size = 109400, upload-time = "2026-07-20T02:05:14.821Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6a/e94133d4c2d1a14d2384310bf3e79d9cf32c9d1eae1c6f034fb80d098fa1/yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077", size = 115934, upload-time = "2026-07-20T02:05:17.78Z" }, + { url = "https://files.pythonhosted.org/packages/4e/3c/34955ed967b976fc38edcbb6d538dee79dbda4cb7fc7f72a0907a7c78e0f/yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd", size = 112178, upload-time = "2026-07-20T02:05:19.675Z" }, + { url = "https://files.pythonhosted.org/packages/f5/46/d7bd3a8859d47dcfaffd7127af7076032a7da278a9a02e17b5f37bfb6712/yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25", size = 97544, upload-time = "2026-07-20T02:05:21.523Z" }, + { url = "https://files.pythonhosted.org/packages/01/69/c1bfd21e32c638974ea2c542a0b8c53ef1fa9eff336020f5d014f9503ff2/yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a", size = 93359, upload-time = "2026-07-20T02:05:23.493Z" }, + { url = "https://files.pythonhosted.org/packages/1b/84/71d051c850b5af41d168c679d9eb67eb7c55283ac4ee131673edf134bc4e/yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d", size = 136035, upload-time = "2026-07-20T02:05:25.489Z" }, + { url = "https://files.pythonhosted.org/packages/03/4d/8ad27f9a1b7e69313cca5d695b925b48efe51208d3490e0844bae97cabc0/yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec", size = 97642, upload-time = "2026-07-20T02:05:27.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b4/05b4131c407006cd1e410e9c6539f16a0945724677e5364447313c15ea3e/yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c", size = 97323, upload-time = "2026-07-20T02:05:29.441Z" }, + { url = "https://files.pythonhosted.org/packages/20/16/e618c875c73e0e39611f20a581b3d5e8d59b8857bf001bee3263044c6deb/yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54", size = 107741, upload-time = "2026-07-20T02:05:31.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/9a/c4defeaf3ed33fcb346aacf9c6e971a8d4e2bde04a0310e79abb208e7965/yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12", size = 103570, upload-time = "2026-07-20T02:05:33.303Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e7/0e0e0de5865ebd5914537ef486f36c727a59865c3ac0cf5ff1b32aececbf/yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d", size = 115815, upload-time = "2026-07-20T02:05:35.292Z" }, + { url = "https://files.pythonhosted.org/packages/2b/27/ca56b700cb170aba25a3893b75355b213935657dc5714d2383354a270e62/yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1", size = 116025, upload-time = "2026-07-20T02:05:37.503Z" }, + { url = "https://files.pythonhosted.org/packages/d6/d0/d56c859b8222116f5d68459199f48359e0bf121b6f65a69bf329b3602ba0/yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9", size = 109835, upload-time = "2026-07-20T02:05:39.506Z" }, + { url = "https://files.pythonhosted.org/packages/70/a2/3a35557e4d1a79425040eba202ccaf08bdc8717680fc77e2498a1ad2e0a5/yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027", size = 108884, upload-time = "2026-07-20T02:05:41.584Z" }, + { url = "https://files.pythonhosted.org/packages/e4/35/ef4c26356b7913c68983bac2d72a4212b3347af551cb8d250b99b5ed7b7f/yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b", size = 107308, upload-time = "2026-07-20T02:05:43.697Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/ff0dc66c2ccf3e0153ab97ff61eabab4400e6a5264af427ab30cd69f1857/yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293", size = 103646, upload-time = "2026-07-20T02:05:45.895Z" }, + { url = "https://files.pythonhosted.org/packages/74/f0/33b9271c7f881766359d58266fa0811d2e5210ed860e28da7dc6d7786344/yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e", size = 115305, upload-time = "2026-07-20T02:05:47.832Z" }, + { url = "https://files.pythonhosted.org/packages/ef/65/fd79fb1868c4a80db8661091de525bf430f63c3bea1b20e8b6a84fc7d359/yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b", size = 108404, upload-time = "2026-07-20T02:05:49.604Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ba/dbabe6b262f17a816c70cfc09558dbf03ece3ec76684d02f911a3d3a189c/yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce", size = 115940, upload-time = "2026-07-20T02:05:51.741Z" }, + { url = "https://files.pythonhosted.org/packages/a5/43/fab2d1dad9d340a268cdde63756a123d069723efff6a372d123fa74a9517/yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba", size = 110006, upload-time = "2026-07-20T02:05:53.554Z" }, + { url = "https://files.pythonhosted.org/packages/c4/27/41eb51bbd1b8d89546b83897cfb0164f1e109304fd408dbb151b639eec0f/yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b", size = 97618, upload-time = "2026-07-20T02:05:55.57Z" }, + { url = "https://files.pythonhosted.org/packages/3c/25/b2553764b3d65db711d8f45416351ec4f420847558eb669edcbcaadf5780/yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c", size = 93018, upload-time = "2026-07-20T02:05:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/e1/63/64ef361967cc983573149dc1515d531db5da8a4c92d22bb833d59e01b313/yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2", size = 135075, upload-time = "2026-07-20T02:05:59.671Z" }, + { url = "https://files.pythonhosted.org/packages/bb/89/55920fd853ce43e608adbc3962456f0d649d6bb15250dc2988321da0fe1c/yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb", size = 97225, upload-time = "2026-07-20T02:06:01.769Z" }, + { url = "https://files.pythonhosted.org/packages/15/f0/7688d3f2cfff7590df2af38ec46d969f4281a4dddb08a9ad2eafbcdddf98/yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075", size = 96751, upload-time = "2026-07-20T02:06:03.676Z" }, + { url = "https://files.pythonhosted.org/packages/05/1a/a851a0f94aaaf379dd4f901bfc80f634280bec51eb260b47363e2a4cd62e/yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff", size = 107960, upload-time = "2026-07-20T02:06:05.699Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a8/faea066c12f9c77ca0de90641f1655f9dd7b412477bf28c76d692f3aecff/yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448", size = 103500, upload-time = "2026-07-20T02:06:07.556Z" }, + { url = "https://files.pythonhosted.org/packages/fb/9c/1e67084c2a6e2f2db0e3be798328cb3be42c0119b621d25461479a224d21/yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f", size = 115780, upload-time = "2026-07-20T02:06:09.599Z" }, + { url = "https://files.pythonhosted.org/packages/58/86/1f94664e147474337e3359f52012cf3d02f825f694317b178bfba1078c62/yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd", size = 115308, upload-time = "2026-07-20T02:06:11.352Z" }, + { url = "https://files.pythonhosted.org/packages/0a/43/8e55ae7538ba5f28ccb3c845c6dd4549cf7016d5992e5326512519107cdd/yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16", size = 110574, upload-time = "2026-07-20T02:06:13.129Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ba/a889ec8765cedcf2ac44dcb02d6a21e4861399b243b263c5f2dde27ee740/yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213", size = 109914, upload-time = "2026-07-20T02:06:15.243Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c3/e45f821af67b791c2dbbe4a9f4137a1d33f8d386654a05a0c3f47bdfa25d/yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24", size = 107712, upload-time = "2026-07-20T02:06:17.443Z" }, + { url = "https://files.pythonhosted.org/packages/02/00/2ab0f42c9857fcb490bfaa6647b14540b53d241ab209f23220b958cc5832/yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385", size = 104251, upload-time = "2026-07-20T02:06:19.259Z" }, + { url = "https://files.pythonhosted.org/packages/7a/70/709d9a286e98af2c7fd8e4e6cada658b5c0e30d87dd7e2a63c2fb5767217/yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c", size = 115319, upload-time = "2026-07-20T02:06:21.207Z" }, + { url = "https://files.pythonhosted.org/packages/5c/6c/3eaa515142991fe84cfc483ff986492211f1978f90161ccefdbec919d09b/yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4", size = 109163, upload-time = "2026-07-20T02:06:23.006Z" }, + { url = "https://files.pythonhosted.org/packages/bb/64/711dafce66c323a3144d470547a71c5384c57623308ac8bb5e4b903ac148/yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144", size = 115435, upload-time = "2026-07-20T02:06:24.923Z" }, + { url = "https://files.pythonhosted.org/packages/cf/f3/9b9d0e6d84bea851eb1ba99e4bdc755b86fd813e49ec86dfe42f26befdef/yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4", size = 110691, upload-time = "2026-07-20T02:06:26.973Z" }, + { url = "https://files.pythonhosted.org/packages/86/e4/62a06b7e87c4246ac76b7c2da136f972eb4a3a1fc94abb07e7022d6fdb0a/yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740", size = 97454, upload-time = "2026-07-20T02:06:29.163Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c9/5fc8025b318ab10db413b61056bd0d95c557a70e8df4210c7511f866329c/yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1", size = 92813, upload-time = "2026-07-20T02:06:31.113Z" }, + { url = "https://files.pythonhosted.org/packages/a9/08/5f3085fef9564217074db9dd8573de1795bc82cde61a7ad10b6a7234a569/yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76", size = 135680, upload-time = "2026-07-20T02:06:33.273Z" }, + { url = "https://files.pythonhosted.org/packages/98/35/ba9436e579bd48a8801f2021d842d9ab4994c26e4c7dd3a4c1f1bcb57a9e/yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d", size = 97395, upload-time = "2026-07-20T02:06:35.259Z" }, + { url = "https://files.pythonhosted.org/packages/18/a9/a07f76f3c44e02b25cc743af5ef93eef27f7013eadca770451b6a6ccb5db/yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75", size = 97223, upload-time = "2026-07-20T02:06:37.216Z" }, + { url = "https://files.pythonhosted.org/packages/77/f7/a9a1d6fa7dd9e388f95b30f6ad3ec4e285f6c8f61f44ce16070c3fcfe414/yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9", size = 108777, upload-time = "2026-07-20T02:06:39.292Z" }, + { url = "https://files.pythonhosted.org/packages/2f/44/e0b86c302471fabd6f02808ecf2ac52b8412b624787849d4bf2cdb466f6f/yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede", size = 103119, upload-time = "2026-07-20T02:06:41.456Z" }, + { url = "https://files.pythonhosted.org/packages/d1/16/9c16d180bf8faaf223225eb50e1245870ff1ae0e302a27153988e65c51fd/yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca", size = 116471, upload-time = "2026-07-20T02:06:43.696Z" }, + { url = "https://files.pythonhosted.org/packages/d2/8d/b219b9df28a02ce95cfbdd41d2f7caa5669d0ff979c1c9975697145e33c5/yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027", size = 115974, upload-time = "2026-07-20T02:06:45.874Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e8/f20557aca240d88e69850ad1ee91756821d094bb1310565c04d25c6682a2/yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9", size = 110830, upload-time = "2026-07-20T02:06:47.852Z" }, + { url = "https://files.pythonhosted.org/packages/db/18/199b85109a53eeca64ee19c9cca228287e8e4ab0cc1a09b28f530e65cce0/yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41", size = 110054, upload-time = "2026-07-20T02:06:49.84Z" }, + { url = "https://files.pythonhosted.org/packages/aa/2f/ed28147f8cd7f48c49367c90713b30a555284b6105a6a56f3a05568da795/yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373", size = 108312, upload-time = "2026-07-20T02:06:51.835Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c5/55e16ae0a5c227cea8df1c6871ba57d614a34243146c05729caf2a1bd9c5/yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36", size = 103662, upload-time = "2026-07-20T02:06:54.061Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ea/dbd7c2caec459c9a426f18b02688ecbfb58620d0f6a3422d24769fbaf8ab/yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0", size = 116090, upload-time = "2026-07-20T02:06:56.015Z" }, + { url = "https://files.pythonhosted.org/packages/06/84/39ce4ce3059e07fece5fbdbee8c4053406af9aca911ce9fa5f8548aab6af/yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5", size = 109523, upload-time = "2026-07-20T02:06:57.926Z" }, + { url = "https://files.pythonhosted.org/packages/a9/8b/71ff44137b405c64a7788075669c24010019f57a7464b78c3a6cbee539d9/yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5", size = 116084, upload-time = "2026-07-20T02:06:59.868Z" }, + { url = "https://files.pythonhosted.org/packages/62/c0/423078fdd4042e1862c11f0ffd977a0ffa393783c12bee94685923bc189e/yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4", size = 111006, upload-time = "2026-07-20T02:07:01.907Z" }, + { url = "https://files.pythonhosted.org/packages/cf/52/6daa2ee9d95e5c98b8128f8df91eb692eb423ab274b8cf08db52152fad26/yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad", size = 99215, upload-time = "2026-07-20T02:07:03.852Z" }, + { url = "https://files.pythonhosted.org/packages/ec/0e/464a847d7359e0da75dd9fc5c1d1aa35d0159ea31e5f8e66a3c1c29ff3d0/yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f", size = 94566, upload-time = "2026-07-20T02:07:06.074Z" }, + { url = "https://files.pythonhosted.org/packages/e2/55/e03acc4446772660bc335e86e41ef31e4d0d838fd641531a11a5ee33b493/yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88", size = 142533, upload-time = "2026-07-20T02:07:08.284Z" }, + { url = "https://files.pythonhosted.org/packages/ae/71/4acd3a1fc7cf14345cdb302665ecd2097f62c365b4f14ca17d4f37775cf9/yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba", size = 100776, upload-time = "2026-07-20T02:07:10.197Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0b/cfb76b7fe99686db264bff829779a539d923e7564ffd7ef18da6c54c3774/yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928", size = 100913, upload-time = "2026-07-20T02:07:12.357Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3f/7116e782992abbd4fb6948488aec72078895e929a23078290739e8396fce/yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f", size = 106507, upload-time = "2026-07-20T02:07:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/33/90/d4d2d73ee78229cc889872eb8e085d8f5c6f51abdb178409fd9b23cf74fd/yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95", size = 99219, upload-time = "2026-07-20T02:07:16.019Z" }, + { url = "https://files.pythonhosted.org/packages/3e/fa/a6df1a9bccd644eec00abee0dff4277416222cec435330fd1f2858523ec1/yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc", size = 111804, upload-time = "2026-07-20T02:07:18.141Z" }, + { url = "https://files.pythonhosted.org/packages/8a/9e/7b2a1f4bcc20e9447156dd2b1c4d01f70d9df0759025ee7d09a84ffae134/yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da", size = 110943, upload-time = "2026-07-20T02:07:20.06Z" }, + { url = "https://files.pythonhosted.org/packages/08/ff/22c92affb0f9b623ca753d27d968b5625b868f12c6378d049d55ae247643/yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a", size = 108251, upload-time = "2026-07-20T02:07:22.217Z" }, + { url = "https://files.pythonhosted.org/packages/45/44/5769b96298c1e195fb412997b6090af2a84105cf59c17613558a2d011d1f/yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0", size = 106025, upload-time = "2026-07-20T02:07:24.083Z" }, + { url = "https://files.pythonhosted.org/packages/4c/40/009e8e791fd9762c0e1567e69248acb4f49064597e1680874c16dd8bb798/yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498", size = 106573, upload-time = "2026-07-20T02:07:26.248Z" }, + { url = "https://files.pythonhosted.org/packages/20/c6/b7480578f8a0a80946f36ad6df547ecec704f9ba69d2de60f8aa6f1c1cbf/yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104", size = 100751, upload-time = "2026-07-20T02:07:28.098Z" }, + { url = "https://files.pythonhosted.org/packages/d4/27/4476f3360b91a48c5cf125e91f59a3bd35299d84a431a258d57f5977bb11/yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331", size = 111643, upload-time = "2026-07-20T02:07:30.88Z" }, + { url = "https://files.pythonhosted.org/packages/4c/4b/5cdd3e5ee944e8af31e52f6cd3d3af5fd7b937e036ccbbba2c9ffebede95/yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550", size = 106312, upload-time = "2026-07-20T02:07:33.06Z" }, + { url = "https://files.pythonhosted.org/packages/18/86/f406b0c2a6f99575de2da671ef47aa06f89a5be83a27a46971c3b86cecdb/yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6", size = 110379, upload-time = "2026-07-20T02:07:35.155Z" }, + { url = "https://files.pythonhosted.org/packages/f0/6c/9f3adfbd3b30b4fa0f7ccb3a83eba2c1152d3fff554d535e640ba0f7ba2b/yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047", size = 108497, upload-time = "2026-07-20T02:07:37.35Z" }, + { url = "https://files.pythonhosted.org/packages/dd/37/91eb2e5ca883a529c1b390348a74cd9fc0512171727f547ce70bfe02be5c/yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104", size = 102450, upload-time = "2026-07-20T02:07:39.578Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f4/ed5c402ac8fde4403ed3366c2716bfddc8a6677ebd59f3d62772cc7fe468/yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688", size = 97222, upload-time = "2026-07-20T02:07:41.55Z" }, + { url = "https://files.pythonhosted.org/packages/61/02/962c1cbfc401a30c1d034dc67ff395f64b52302c6d62de556c1fca99acc0/yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7", size = 58612, upload-time = "2026-07-20T02:07:43.461Z" }, +] + [[package]] name = "zipp" version = "4.1.0" From af0528ab1df69c3ffaef89269cdad50a29477e1a Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 10:11:33 -0500 Subject: [PATCH 05/24] Bench(fix[lgtm]): Check that a published port reaches this container why: The README explained at length that a host process already bound to a port shadows the container's, so a query succeeds against the wrong backend and returns plausible data -- then shipped no way to detect it. Explaining a hazard without supplying the check is the worst of both: the reader knows to worry and has nothing to act on. what: - Add scripts/lgtm/verify.sh, comparing each service's build info as seen from inside the container against the same URL from the host, since liveness cannot tell the two apart but identity can - Run it from up.sh before reporting success, so a shadowed port fails the start rather than surfacing later as a confusing query result - Expose it as just otel-ports and document it - Cover loki, which the ad-hoc checks had been skipping - Drop the LANE alias in the dashboard generator; SCOPE was the only name it needed, and the regenerated JSON is byte-identical - Show just otel-load as a runnable command rather than only naming it in prose --- justfile | 5 +++ scripts/lgtm/README.md | 26 +++++++++++++-- scripts/lgtm/generate_dashboards.py | 9 +++-- scripts/lgtm/up.sh | 7 ++++ scripts/lgtm/verify.sh | 51 +++++++++++++++++++++++++++++ 5 files changed, 90 insertions(+), 8 deletions(-) create mode 100755 scripts/lgtm/verify.sh diff --git a/justfile b/justfile index b77ccfecb..19a269471 100644 --- a/justfile +++ b/justfile @@ -155,6 +155,11 @@ otel-up: otel-down: docker rm -f ${LIBTMUX_LGTM_CONTAINER:-libtmux-lgtm} +# Confirm each published port reaches the container, not a host process +[group: 'otel'] +otel-ports: + scripts/lgtm/verify.sh + # Regenerate the provisioned Grafana dashboards from their generator [group: 'otel'] otel-dashboards: diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md index c8d62c667..3e80d2920 100644 --- a/scripts/lgtm/README.md +++ b/scripts/lgtm/README.md @@ -35,6 +35,12 @@ Drive a workload: $ just otel-smoke ``` +Drive a ramping arrival rate instead, to find where a transport saturates: + +```console +$ just otel-load +``` + Check that every panel has data: ```console @@ -65,6 +71,14 @@ reach the wrong server and return plausible data, which costs far more debugging time than a refused connection. Override with `LIBTMUX_LGTM_GRAFANA_PORT` and `LIBTMUX_LGTM_PROM_PORT` if these collide too. +`just otel-up` checks for this before reporting success, by comparing each +service's build info as seen from inside the container against the same URL +from the host. Run it alone at any time: + +```console +$ just otel-ports +``` + ## What the workload emits `scripts/otel_smoke.py` runs every transport — subprocess and control mode, @@ -183,9 +197,15 @@ politely instead of breaking. `just otel-load` uses [rampa](https://github.com/tony/rampa) for the shapes that expose it, notably `ramping-arrival-rate` -- an open model that keeps issuing commands at a target rate whether or not the previous ones finished, so -latency climbs on its own once a transport saturates. The difference is -visible: the same control-mode engine measures p99 around 2 ms under the steady -workload and around 16 ms at the top of the ramp. +latency climbs on its own once a transport saturates: + +```console +$ just otel-load +``` + +The difference is visible: the same control-mode engine measures p99 around +2 ms under the steady workload and around 16 ms at the top of the ramp. Pick a +transport with `LIBTMUX_LOAD_LANE=subprocess-async`. k6 was the obvious alternative and is the wrong tool here. Its value is HTTP load at scale, and there is no HTTP surface in front of the engines -- driving diff --git a/scripts/lgtm/generate_dashboards.py b/scripts/lgtm/generate_dashboards.py index 50a569086..c6bf20cc0 100644 --- a/scripts/lgtm/generate_dashboards.py +++ b/scripts/lgtm/generate_dashboards.py @@ -32,7 +32,6 @@ # question is "did this branch change anything", and leaving it out silently # mixes two branches into one line. SCOPE = 'tmux_lane=~"$lane", vcs_ref_head_name=~"$branch"' -LANE = SCOPE # queries below read naturally as "the current scope" TAGS = ["libtmux", "generated"] BUCKET = "tmux_command_duration_seconds_bucket" @@ -40,7 +39,7 @@ def rate_by(label: str, metric: str) -> str: """Per-second rate of *metric*, grouped by *label*.""" - return f"sum by ({label}) (rate({metric}{{{LANE}}}[$__rate_interval]))" + return f"sum by ({label}) (rate({metric}{{{SCOPE}}}[$__rate_interval]))" def window_total(metric: str, label: str | None = None) -> str: @@ -54,7 +53,7 @@ def window_total(metric: str, label: str | None = None) -> str: in the window the viewer selected, which is both what they meant and immune to staleness. """ - inner = f"increase({metric}{{{LANE}}}[$__range])" + inner = f"increase({metric}{{{SCOPE}}}[$__range])" return f"sum by ({label}) ({inner})" if label else f"sum({inner})" @@ -62,7 +61,7 @@ def window_quantile(quantile: float, label: str) -> str: """Latency *quantile* across the whole window, for summary tables.""" return ( f"histogram_quantile({quantile}, sum by (le, {label}) " - f"(rate({BUCKET}{{{LANE}}}[$__range])))" + f"(rate({BUCKET}{{{SCOPE}}}[$__range])))" ) @@ -70,7 +69,7 @@ def quantile_by(quantile: float, label: str) -> str: """Latency *quantile* from the duration histogram, grouped by *label*.""" return ( f"histogram_quantile({quantile}, sum by (le, {label}) " - f"(rate({BUCKET}{{{LANE}}}[$__rate_interval])))" + f"(rate({BUCKET}{{{SCOPE}}}[$__rate_interval])))" ) diff --git a/scripts/lgtm/up.sh b/scripts/lgtm/up.sh index e37c577ea..983032caa 100755 --- a/scripts/lgtm/up.sh +++ b/scripts/lgtm/up.sh @@ -87,6 +87,13 @@ for _ in $(seq 1 45); do sleep 4 done +# Identity, not liveness: a port that answers may belong to a host process +# that was already bound to it. verify.sh tells the two apart. +"$ROOT/scripts/lgtm/verify.sh" || { + echo "a published port does not reach this container; see the lines above" >&2 + exit 1 +} + cat < /dev/null) + outside=$(curl -s -m 8 "http://127.0.0.1:${host_port}${path}" 2> /dev/null) + if [[ -z "$outside" ]]; then + printf ' %-11s %-6s UNREACHABLE from host\n' "$label" "$host_port" + status=1 + elif [[ "$inside" == "$outside" ]]; then + printf ' %-11s %-6s ok\n' "$label" "$host_port" + else + printf ' %-11s %-6s SHADOWED by another service on this port\n' "$label" "$host_port" + status=1 + fi +} + +check grafana 3000 "$GRAFANA_PORT" /api/health +check prometheus 9090 "$PROM_PORT" /api/v1/status/buildinfo +check tempo 3200 3200 /api/status/buildinfo +check pyroscope 4040 4040 /api/v1/status/buildinfo +check loki 3100 3100 /loki/api/v1/status/buildinfo + +otlp=$(curl -s -o /dev/null -m 8 -w '%{http_code}' -X POST http://127.0.0.1:4318/v1/traces 2> /dev/null) +case "$otlp" in + 2* | 4*) printf ' %-11s %-6s ok\n' otlp-http 4318 ;; + *) + printf ' %-11s %-6s UNREACHABLE (%s)\n' otlp-http 4318 "$otlp" + status=1 + ;; +esac + +exit $status From c8b704040b368b16ff35bd91691d576695bef9ab Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 10:39:07 -0500 Subject: [PATCH 06/24] Bench(fix[lgtm]): Fail fast when the stack is not running why: A closed port is not always refused promptly -- under WSL and inside containers the packets are dropped and the socket waits out its full timeout. With one request per panel target, forgetting `just otel-up` produced a twenty-minute silence and then a report blaming empty panels, which points at the dashboards rather than at the stack. what: - Probe each backend once before checking any panel, and exit naming the fix; the same mistake now costs eighteen seconds and says what to do - Say so in the README, so the guard is discoverable before it fires --- scripts/lgtm/README.md | 5 +++++ scripts/otel_acceptance.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md index 3e80d2920..31a6ddfcc 100644 --- a/scripts/lgtm/README.md +++ b/scripts/lgtm/README.md @@ -232,6 +232,11 @@ Prometheus, Loki, Tempo, or Pyroscope, and fails naming any panel that returned nothing. Because it reads the dashboards themselves, a panel added to the generator is covered the moment it exists. +If the stack is not running the check says so and stops, rather than waiting +out a socket timeout per panel: a closed port is not always refused promptly, +so without that guard a forgotten `just otel-up` becomes a long silence ending +in a confusing report of empty panels. + Ingestion is asynchronous, and each backend buffers on its own schedule, so at any single instant "no data yet" is indistinguishable from "no data ever". The check therefore re-queries the panels that came back empty until they fill or diff --git a/scripts/otel_acceptance.py b/scripts/otel_acceptance.py index 6a1355d0f..cb3c7a031 100644 --- a/scripts/otel_acceptance.py +++ b/scripts/otel_acceptance.py @@ -229,6 +229,36 @@ def check_pyroscope(base: str, profile_type: str, selector: str) -> tuple[bool, return True, f"{len(names)} frames" +def unreachable(endpoints: Endpoints, timeout: float = 5.0) -> list[str]: + """Return the backends that did not answer a health probe. + + Worth doing before any panel is checked. A closed port does not always + refuse a connection promptly -- under WSL and inside containers the packets + are simply dropped, so the socket waits out its full timeout. With one + request per panel target, a stack that is merely not running turns a + mistake into a twenty-minute silence ending in a confusing report about + empty panels. + """ + probes = ( + ("prometheus", f"{endpoints.prometheus}/api/v1/status/buildinfo"), + ("loki", f"{endpoints.loki}/ready"), + ("tempo", f"{endpoints.tempo}/api/echo"), + ("pyroscope", f"{endpoints.pyroscope}/ready"), + ) + + def answers(url: str) -> bool: + try: + with urllib.request.urlopen(url, timeout=timeout): + return True + except urllib.error.HTTPError: + # An HTTP error still proves something is listening and answering. + return True + except (urllib.error.URLError, TimeoutError, OSError): + return False + + return [name for name, url in probes if not answers(url)] + + def check_panel(panel: dict, board: str, endpoints: Endpoints) -> list[Result]: """Verify every target on one panel.""" results: list[Result] = [] @@ -333,6 +363,13 @@ def main(argv: list[str] | None = None) -> int: time.sleep(args.settle) endpoints = Endpoints(args.prometheus, args.loki, args.tempo, args.pyroscope) + + missing = unreachable(endpoints) + if missing: + print(f"these backends did not answer: {', '.join(missing)}", file=sys.stderr) + print("start the stack first: just otel-up", file=sys.stderr) + return 2 + results = check_until(endpoints, args.timeout, args.poll) width = max((len(r.panel) for r in results), default=10) From ef3bd496f15050f8598c697b1564eecfcc96d785 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 10:39:07 -0500 Subject: [PATCH 07/24] Bench(test[lgtm]): Execute the scripts/lgtm doctests why: `scripts` is not in pytest's testpaths, so the doctests under scripts/lgtm were never executed. The examples there carry the load-bearing decision about which facts a metric may keep and which belong on a profile, so they were comments that looked like tests. what: - Run the identity and telemetry doctests from the test suite, skipping the latter when the otel dependency group is absent - Put scripts/lgtm on sys.path from telemetry.py itself, so importing it does not depend on the caller having arranged that first --- scripts/lgtm/telemetry.py | 7 ++++++ tests/test_lgtm_dashboards.py | 45 +++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/scripts/lgtm/telemetry.py b/scripts/lgtm/telemetry.py index 99bf97191..a58a1e0ac 100644 --- a/scripts/lgtm/telemetry.py +++ b/scripts/lgtm/telemetry.py @@ -15,9 +15,16 @@ from __future__ import annotations import contextlib +import pathlib +import sys import time import typing as t +# These modules sit side by side under scripts/, which is not a package and is +# not on sys.path by default. Adding it here means any importer works, rather +# than each caller having to remember. +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + import identity from opentelemetry import baggage, context, metrics, trace from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter diff --git a/tests/test_lgtm_dashboards.py b/tests/test_lgtm_dashboards.py index 71008ecdf..639874b70 100644 --- a/tests/test_lgtm_dashboards.py +++ b/tests/test_lgtm_dashboards.py @@ -11,6 +11,7 @@ import importlib.util import json import pathlib +import sys import typing as t import pytest @@ -241,3 +242,47 @@ def test_every_scope_variable_used_is_defined() -> None: assert variable in names, ( f"{board['uid']} filters on ${variable} without defining it" ) + + +def test_identity_doctests_execute() -> None: + """The doctests documenting the metric/profile split actually run. + + ``scripts`` is not in pytest's testpaths, so nothing else executes them. + A doctest that never runs is a comment that looks like a test, and this + module's examples carry the load-bearing decision: which facts a metric + may keep and which belong on a profile. + """ + import doctest + + spec = importlib.util.spec_from_file_location("identity", _LGTM / "identity.py") + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + results = doctest.testmod(module, verbose=False) + assert results.attempted > 0, "identity.py has no doctests to run" + assert results.failed == 0, f"{results.failed} doctest(s) failed" + + +def test_telemetry_doctests_execute() -> None: + """The same, for the exporter module, when its dependency is installed. + + telemetry.py needs OpenTelemetry, which lives in the ``otel`` dependency + group rather than ``dev`` so the ordinary gates stay lean. Skipping is + honest here; asserting would make the default test run demand a dependency + libtmux itself never imports. + """ + import doctest + + pytest.importorskip("opentelemetry", reason="otel dependency group not installed") + + spec = importlib.util.spec_from_file_location("telemetry", _LGTM / "telemetry.py") + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["telemetry"] = module + spec.loader.exec_module(module) + + results = doctest.testmod(module, verbose=False) + assert results.failed == 0, f"{results.failed} doctest(s) failed" From 51408e14d53b24bdcc7461b8fa16650bfa173ba3 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 10:53:38 -0500 Subject: [PATCH 08/24] Bench(fix[lgtm]): Keep the documentation from drifting off the code why: The README's commands and board names are only correct until someone renames one, and the failure is silent -- the prose still reads well and the command simply does not work. Both sides live in this repo, so the check is cheap and there is no reason to rely on noticing. what: - Add tests asserting every `just` command the README demonstrates is a real recipe, and every generated board is named somewhere in it - Point the instrumentation page at the stack that consumes what it describes; it explained how to export to OTLP without mentioning that an exporter, a Grafana stack, and dashboards already ship here - Drop a fragile test count from that page, which was already stale - Remove a dead assignment in scope() and the unused parameter on the rampa teardown, which is called with none --- docs/experimental/instrumentation.md | 12 +++++++--- scripts/lgtm/load_tmux.py | 8 +++++-- scripts/lgtm/telemetry.py | 1 - tests/test_lgtm_dashboards.py | 35 ++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/docs/experimental/instrumentation.md b/docs/experimental/instrumentation.md index a4bb14449..9743b990f 100644 --- a/docs/experimental/instrumentation.md +++ b/docs/experimental/instrumentation.md @@ -191,6 +191,11 @@ sink exports over OTLP, where `{ span.tmux.commands > 1 }` finds every request that batched work. Nothing else changes, because the sink never learns which engine it is observing. +That exporter already exists, along with a local Grafana stack to receive it and +dashboards built on these counts. `just otel-verify` starts it, drives a tmux +workload through the seam, and checks that every dashboard panel has data. See +`scripts/lgtm/README.md`. + ## Async {func}`~libtmux.experimental.engines.instrumentation.instrument` returns @@ -282,9 +287,10 @@ hooks would build a context object per call whether or not one is used. Wrapping adds one Python call plus each sink's own work, and only for programs that ask for it. -Two tests hold the claim: one asserts wrapping adds no attribute to the engine -it wraps, and one measures an instrumented call against the millisecond scale of -a real tmux round trip. +The claim is held by two tests: one asserts that wrapping adds no attribute to +the engine it wraps, and one measures an instrumented call against the +millisecond scale of a real tmux round trip. + ```console $ uv run pytest tests/experimental/engines/test_instrumentation.py ``` diff --git a/scripts/lgtm/load_tmux.py b/scripts/lgtm/load_tmux.py index 4c5a1b8c9..705a5dfd2 100644 --- a/scripts/lgtm/load_tmux.py +++ b/scripts/lgtm/load_tmux.py @@ -163,8 +163,12 @@ async def ramp(worker: rampa.Worker) -> None: await _issue(worker, PLAIN) -async def teardown(_: t.Any = None) -> None: - """Close the engine and remove the tmux server.""" +async def teardown() -> None: + """Close the engine and remove the tmux server. + + rampa calls this with no arguments once the run finishes, and reports a + failure here as ``TEARDOWN_FAILED`` rather than swallowing it. + """ engine = _STATE.get("engine") if engine is not None and hasattr(engine.inner, "aclose"): await engine.inner.aclose() diff --git a/scripts/lgtm/telemetry.py b/scripts/lgtm/telemetry.py index a58a1e0ac..37fdf3f42 100644 --- a/scripts/lgtm/telemetry.py +++ b/scripts/lgtm/telemetry.py @@ -112,7 +112,6 @@ def scope(**entries: str) -> t.Iterator[None]: >>> with scope(**{"libtmux.phase": "warmup"}): ... pass """ - token = None current = context.get_current() for key, value in entries.items(): current = baggage.set_baggage(key, value, context=current) diff --git a/tests/test_lgtm_dashboards.py b/tests/test_lgtm_dashboards.py index 639874b70..5e0828aef 100644 --- a/tests/test_lgtm_dashboards.py +++ b/tests/test_lgtm_dashboards.py @@ -286,3 +286,38 @@ def test_telemetry_doctests_execute() -> None: results = doctest.testmod(module, verbose=False) assert results.failed == 0, f"{results.failed} doctest(s) failed" + + +def test_readme_only_shows_commands_that_exist() -> None: + """Every ``just`` command the README demonstrates is a real recipe. + + A renamed recipe is the likeliest way this documentation rots, and the + failure is silent: the prose still reads correctly and the command simply + does not work. Checking is cheap because both sides are in the repo. + """ + import re + + readme = (_LGTM / "README.md").read_text(encoding="utf-8") + justfile = (_ROOT / "justfile").read_text(encoding="utf-8") + + shown = set(re.findall(r"^\$ just ([a-z][a-z-]*)", readme, re.MULTILINE)) + assert shown, "the README stopped showing any just commands" + defined = set( + re.findall(r"^([a-z][a-z-]*)(?: \*?[a-z_]+)?:", justfile, re.MULTILINE) + ) + missing = shown - defined + assert not missing, f"README shows recipes that do not exist: {sorted(missing)}" + + +def test_readme_names_every_dashboard_it_ships() -> None: + """The README's board list matches the boards actually generated. + + Adding a board and forgetting to mention it leaves it undiscoverable, + which for a dashboard is the same as not shipping it. + """ + readme = (_LGTM / "README.md").read_text(encoding="utf-8") + for path in sorted(_DASHBOARDS.glob("*.json")): + board = json.loads(path.read_text(encoding="utf-8")) + assert board["title"] in readme, ( + f"{board['title']} is generated but never mentioned in the README" + ) From b26600287fd4fe38ee3cbc33c0021480a95a5667 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 11:09:11 -0500 Subject: [PATCH 09/24] Bench(fix[lgtm]): Make the log-to-trace link real why: The stack advertised that a log line links to the trace it came from, and the datasource was wired for it, but no log record carried a trace id. A record only picks up trace context while a span is current, and the lane summaries were logged between lanes, with no span open. The claim was false and nothing checked it. what: - Log each lane's result from inside a short span, so the record carries a trace id and the Loki-to-Tempo jump works; verified by taking a trace id from a Loki line and resolving it to its span in Tempo - Put the lane's totals on that span too, so the trace answers the same question the log line does --- scripts/otel_smoke.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/scripts/otel_smoke.py b/scripts/otel_smoke.py index 5c89475a1..dcde3c4ec 100644 --- a/scripts/otel_smoke.py +++ b/scripts/otel_smoke.py @@ -140,6 +140,21 @@ async def worker() -> None: await asyncio.gather(*(worker() for _ in range(concurrency))) +def _log_lane(signals: t.Any, lane: str, totals: dict[str, int]) -> None: + """Log a lane's result from inside a span, so the line links to a trace. + + A log record only carries trace context when a span is current; emitted + between lanes it would reach Loki with the run's identity but nothing to + click through to. Opening a short span around the record is what makes the + Loki-to-Tempo jump in the datasource configuration actually work. + """ + with signals.tracer.start_as_current_span("tmux lane summary") as span: + span.set_attribute("tmux.lane", lane) + for key, value in totals.items(): + span.set_attribute(f"tmux.{key}", value) + logger.info("lane finished", extra={"lane": lane, **totals}) + + def lane_totals(counts: CountingSink) -> dict[str, int]: """Summarize one lane's locally observed counts.""" return { @@ -226,7 +241,7 @@ def main(argv: list[str] | None = None) -> int: with telemetry.scope(**{"libtmux.phase": f"{lane}-sync"}): run_sync(engine, args.seconds) totals[lane] = lane_totals(counts) - logger.info("lane finished", extra={"lane": lane, **totals[lane]}) + _log_lane(signals, lane, totals[lane]) async_lanes = ( ("subprocess-async", lambda: AsyncSubprocessEngine.for_server(server)), @@ -253,7 +268,7 @@ async def drive(engine: t.Any = engine) -> None: with telemetry.scope(**{"libtmux.phase": f"{lane}-async"}): asyncio.run(drive()) totals[lane] = lane_totals(counts) - logger.info("lane finished", extra={"lane": lane, **totals[lane]}) + _log_lane(signals, lane, totals[lane]) finally: subprocess.run( ("tmux", "-S", str(root / "smoke.sock"), "kill-server"), From c2c51e5dc86ebfceedfac00c16471b0e8229f8dd Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 11:35:02 -0500 Subject: [PATCH 10/24] Bench(test[lgtm]): Pin the identity rules the docs promise why: The resolver's edge cases carried the load and none were tested. A detached checkout answers the literal string HEAD, so used directly every detached run on every branch would share one dimension value and could not be told apart; CI needs its explicit ref to win over the checkout's; and a directory outside git has to yield nothing rather than raise. All three were verified by hand and nothing kept them true. what: - Test the ref override, the detached-HEAD fallback to tag then short revision, and the empty result outside a repository, each against a real git repository rather than a fixture that happens to exist - Correct the latency bucket comment, which claimed control mode costs tens of microseconds; the measured median is a couple of hundred. The buckets were already right and still resolve every lane's median into its own band --- scripts/lgtm/telemetry.py | 8 ++- tests/test_lgtm_dashboards.py | 101 ++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 3 deletions(-) diff --git a/scripts/lgtm/telemetry.py b/scripts/lgtm/telemetry.py index 37fdf3f42..0cce5da5a 100644 --- a/scripts/lgtm/telemetry.py +++ b/scripts/lgtm/telemetry.py @@ -45,9 +45,11 @@ SERVICE_NAME = "libtmux-engines" -# Latency buckets in seconds. A control-mode command is tens of microseconds and -# a subprocess command is a few milliseconds, so the buckets have to span four -# orders of magnitude or one transport lands entirely in the first bucket. +# Latency buckets in seconds. Measured medians run from a couple of hundred +# microseconds on a persistent control-mode client to a few milliseconds per +# subprocess, and tails reach far past both, so the buckets have to span four +# orders of magnitude or a transport lands entirely in the first one and its +# percentiles say nothing. DURATION_BUCKETS = ( 0.0001, 0.00025, diff --git a/tests/test_lgtm_dashboards.py b/tests/test_lgtm_dashboards.py index 5e0828aef..19b86ae03 100644 --- a/tests/test_lgtm_dashboards.py +++ b/tests/test_lgtm_dashboards.py @@ -8,9 +8,11 @@ from __future__ import annotations +import functools import importlib.util import json import pathlib +import subprocess import sys import typing as t @@ -321,3 +323,102 @@ def test_readme_names_every_dashboard_it_ships() -> None: assert board["title"] in readme, ( f"{board['title']} is generated but never mentioned in the README" ) + + +def _identity() -> t.Any: + """Import the identity resolver from ``scripts/``.""" + spec = importlib.util.spec_from_file_location("identity", _LGTM / "identity.py") + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _git_repo(root: pathlib.Path) -> None: + """Create a one-commit git repository at *root*.""" + run = functools.partial(subprocess.run, cwd=root, check=True, capture_output=True) + run(["git", "init", "-q", "-b", "trunk"]) + run(["git", "config", "user.email", "t@example.invalid"]) + run(["git", "config", "user.name", "Test"]) + (root / "f.txt").write_text("x", encoding="utf-8") + run(["git", "add", "f.txt"]) + run(["git", "commit", "-qm", "first"]) + + +def test_an_explicit_ref_beats_the_checkout(tmp_path: pathlib.Path) -> None: + """CI checks out a detached HEAD but knows the branch the work belongs to. + + Without the override every CI run would be labelled by its revision, and + comparing a branch against another would be impossible in exactly the + place it matters most. + """ + identity = _identity() + _git_repo(tmp_path) + + resolved = identity.resolve( + run_id="r", + root=tmp_path, + env={"LIBTMUX_VCS_REF": "release/1.2", "LIBTMUX_WORKTREE": "ci-runner"}, + ) + + assert resolved["vcs.ref.head.name"] == "release/1.2" + assert resolved["libtmux.worktree"] == "ci-runner" + assert identity.metric_attributes(resolved)["vcs_ref_head_name"] == "release/1.2" + + +def test_a_detached_head_never_labels_itself_head(tmp_path: pathlib.Path) -> None: + """Detached checkouts must not collapse onto one meaningless label. + + ``git rev-parse --abbrev-ref HEAD`` answers the literal string ``HEAD`` + when detached. Used directly, every detached run on every branch would + share one dimension value and could not be told apart. + """ + identity = _identity() + _git_repo(tmp_path) + subprocess.run( + ["git", "checkout", "-q", "--detach"], + cwd=tmp_path, + check=True, + capture_output=True, + ) + + attributes = identity.vcs_attributes(tmp_path) + + assert attributes["vcs.ref.head.name"] != "HEAD" + assert attributes["vcs.ref.head.type"] in {"tag", "revision"} + assert attributes["vcs.ref.head.revision"].startswith( + attributes["vcs.ref.head.name"] + ) + + +def test_a_tagged_detached_head_reports_the_tag(tmp_path: pathlib.Path) -> None: + """A tag is more meaningful than a revision, so it wins when present.""" + identity = _identity() + _git_repo(tmp_path) + subprocess.run( + ["git", "tag", "v9.9.9"], cwd=tmp_path, check=True, capture_output=True + ) + subprocess.run( + ["git", "checkout", "-q", "--detach"], + cwd=tmp_path, + check=True, + capture_output=True, + ) + + attributes = identity.vcs_attributes(tmp_path) + + assert attributes["vcs.ref.head.name"] == "v9.9.9" + assert attributes["vcs.ref.head.type"] == "tag" + + +def test_a_directory_outside_git_yields_no_vcs_attributes( + tmp_path: pathlib.Path, +) -> None: + """Telemetry must still run where there is no repository to describe.""" + identity = _identity() + + assert identity.vcs_attributes(tmp_path) == {} + resolved = identity.resolve(run_id="r", root=tmp_path, env={}) + assert resolved["libtmux.run_id"] == "r" + assert "vcs.ref.head.name" not in resolved From 4c86d43dffc3af1f7f382070b16c7b00b9c150b3 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 14:04:05 -0500 Subject: [PATCH 11/24] Bench(fix[lgtm]): Attempt the load setup once, not once per iteration why: A single unknown lane name saturated the machine. Building the scenario's state costs a tmux server and three exporter threads, and it was built before the step that failed, so every retry left another set behind: 3,539 tmux servers, 10,619 threads, a load average near 2,000, and a test suite that then failed 24 tests for reasons that had nothing to do with the code. rampa is not at fault. Isolating a failing iteration and running the next one is what a load tool should do, and a minimal scenario that raises on every iteration holds a flat three threads. Reproducing it took twenty lines: build three threads per iteration, then raise, and a six-second run reaches a thousand threads. The defect is a scenario that rebuilds expensive state on a path that cannot succeed. what: - Latch a setup failure and re-raise it, so an impossible setup is attempted once however many iterations follow - Validate the lane at import, before a tmux server exists, so a typo costs three seconds and names the valid lanes - Test both: that fifty iterations produce one setup attempt, and that the lane check precedes anything that creates --- scripts/lgtm/load_tmux.py | 65 +++++++++++++++++++++++++---------- scripts/otel_smoke.py | 30 ++++++++++++++-- tests/test_lgtm_dashboards.py | 56 ++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 20 deletions(-) diff --git a/scripts/lgtm/load_tmux.py b/scripts/lgtm/load_tmux.py index 705a5dfd2..a0846b538 100644 --- a/scripts/lgtm/load_tmux.py +++ b/scripts/lgtm/load_tmux.py @@ -40,7 +40,24 @@ from libtmux.experimental.engines.control_mode import command_count from libtmux.server import Server +# Transports this scenario can drive, and how to build each. +LANES = { + "control-async": AsyncControlModeEngine.for_server, + "subprocess-async": AsyncSubprocessEngine.for_server, +} + LANE = os.environ.get("LIBTMUX_LOAD_LANE", "control-async") +if LANE not in LANES: + # Validated at import, before any tmux server exists. Left until the first + # worker iteration, an unknown lane raises inside a rampa worker, which + # counts the iteration as failed and moves on -- so the next iteration + # builds another tmux server, and the next, for the whole run. A single + # typo produced thousands of servers before this check existed. + message = ( + f"LIBTMUX_LOAD_LANE={LANE!r} is not a lane; choose one of " + f"{', '.join(sorted(LANES))}" + ) + raise SystemExit(message) PLAIN = CommandRequest.from_args("list-panes", "-a", "-F", "#{pane_id}") GROUPED = CommandRequest.from_args( "set-option", @@ -104,27 +121,39 @@ def _engine() -> t.Any: spike labels as every other run, so the two are directly comparable instead of arriving as a second, parallel account of the same work. """ + # A failure here is permanent for this process, so it is remembered and + # re-raised. Building this costs a tmux server and three exporter threads, + # and rampa correctly isolates a failing iteration and runs the next one -- + # so without the latch, a setup that cannot succeed is retried for the whole + # run and leaves a fresh set of resources behind every time. That is what + # once turned one bad lane name into thousands of tmux servers and ten + # thousand threads; the executor was behaving properly, the scenario was not. + if "error" in _STATE: + raise _STATE["error"] if "engine" not in _STATE: os.environ.pop("TMUX", None) os.environ.pop("TMUX_PANE", None) - server, root = _server() - _STATE["root"] = root - signals = telemetry.build( - os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:4318"), - run_id=os.environ.get("LIBTMUX_RUN_ID", f"load-{uuid.uuid4().hex[:8]}"), - spike=os.environ.get("LIBTMUX_SPIKE"), - ) - _STATE["signals"] = signals - factory = { - "control-async": AsyncControlModeEngine.for_server, - "subprocess-async": AsyncSubprocessEngine.for_server, - }[LANE] - _STATE["engine"] = instrument( - factory(server), - telemetry.OTelSink( - signals.tracer, signals.meter, LANE, signals.metric_labels - ), - ) + try: + # Resolve the factory before anything is created, so a failure here + # cannot leave a tmux server behind. + factory = LANES[LANE] + server, root = _server() + _STATE["root"] = root + signals = telemetry.build( + os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:4318"), + run_id=os.environ.get("LIBTMUX_RUN_ID", f"load-{uuid.uuid4().hex[:8]}"), + spike=os.environ.get("LIBTMUX_SPIKE"), + ) + _STATE["signals"] = signals + _STATE["engine"] = instrument( + factory(server), + telemetry.OTelSink( + signals.tracer, signals.meter, LANE, signals.metric_labels + ), + ) + except BaseException as error: + _STATE["error"] = error + raise return _STATE["engine"] diff --git a/scripts/otel_smoke.py b/scripts/otel_smoke.py index dcde3c4ec..e358e4fdb 100644 --- a/scripts/otel_smoke.py +++ b/scripts/otel_smoke.py @@ -17,6 +17,7 @@ import argparse import asyncio +import contextlib import logging import os import pathlib @@ -140,6 +141,25 @@ async def worker() -> None: await asyncio.gather(*(worker() for _ in range(concurrency))) +@contextlib.contextmanager +def _profile_lane(lane: str, *, enabled: bool) -> t.Iterator[None]: + """Tag the CPU profile with the lane running inside the block. + + Pyroscope's configure-time tags describe the whole process, which is fine + for branch or run identity but leaves the flamegraph unable to answer the + question the other signals can: what did *this* transport cost. A dynamic + tag around each lane makes the profile filterable the same way the metrics + and traces already are. + """ + if not enabled: + yield + return + import pyroscope + + with pyroscope.tag_wrapper({"tmux_lane": lane}): + yield + + def _log_lane(signals: t.Any, lane: str, totals: dict[str, int]) -> None: """Log a lane's result from inside a span, so the line links to a trace. @@ -238,7 +258,10 @@ def main(argv: list[str] | None = None) -> int: signals.tracer, signals.meter, lane, signals.metric_labels ), ) - with telemetry.scope(**{"libtmux.phase": f"{lane}-sync"}): + with ( + telemetry.scope(**{"libtmux.phase": f"{lane}-sync"}), + _profile_lane(lane, enabled=profiling), + ): run_sync(engine, args.seconds) totals[lane] = lane_totals(counts) _log_lane(signals, lane, totals[lane]) @@ -265,7 +288,10 @@ async def drive(engine: t.Any = engine) -> None: if hasattr(inner, "aclose"): await inner.aclose() - with telemetry.scope(**{"libtmux.phase": f"{lane}-async"}): + with ( + telemetry.scope(**{"libtmux.phase": f"{lane}-async"}), + _profile_lane(lane, enabled=profiling), + ): asyncio.run(drive()) totals[lane] = lane_totals(counts) _log_lane(signals, lane, totals[lane]) diff --git a/tests/test_lgtm_dashboards.py b/tests/test_lgtm_dashboards.py index 19b86ae03..88bad11e5 100644 --- a/tests/test_lgtm_dashboards.py +++ b/tests/test_lgtm_dashboards.py @@ -422,3 +422,59 @@ def test_a_directory_outside_git_yields_no_vcs_attributes( resolved = identity.resolve(run_id="r", root=tmp_path, env={}) assert resolved["libtmux.run_id"] == "r" assert "vcs.ref.head.name" not in resolved + + +def test_a_failing_load_setup_is_attempted_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Setup that cannot succeed must not be retried by every iteration. + + rampa isolates a failing iteration and runs the next one, which is what a + load tool should do. That makes it the scenario's job not to rebuild + expensive state on a path that always fails: building it costs a tmux + server and three exporter threads, so retrying per iteration turns one bad + input into thousands of servers and enough threads to saturate the machine. + It did exactly that once, which is why this is pinned. + """ + pytest.importorskip("opentelemetry", reason="otel dependency group not installed") + + monkeypatch.syspath_prepend(str(_LGTM)) + spec = importlib.util.spec_from_file_location("load_tmux", _LGTM / "load_tmux.py") + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["load_tmux"] = module + spec.loader.exec_module(module) + + attempts = 0 + + def refuse() -> t.NoReturn: + nonlocal attempts + attempts += 1 + message = "tmux unavailable" + raise RuntimeError(message) + + monkeypatch.setattr(module, "_server", refuse) + + raised = [] + for _ in range(50): + with pytest.raises(RuntimeError, match="tmux unavailable") as caught: + module._engine() + raised.append(caught.value) + + assert attempts == 1, f"setup was retried {attempts} times" + assert all(error is raised[0] for error in raised) + + +def test_an_unknown_load_lane_is_rejected_before_anything_is_built() -> None: + """A typo must fail at import, not once per iteration. + + Validating late means the tmux server is already created by the time the + lane is looked up, and the failure repeats for the whole run. + """ + source = (_LGTM / "load_tmux.py").read_text(encoding="utf-8") + + validation = source.index("if LANE not in LANES:") + creation = source.index("def _server(") + assert validation < creation, "the lane must be validated before _server is defined" + assert "raise SystemExit(message)" in source From 71ecdb14829d6f50e5770dcf0dbade30fde5bc26 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 14:19:25 -0500 Subject: [PATCH 12/24] Bench(feat[lgtm]): Let an agent query the stack over MCP why: Both Grafana and Tempo expose MCP servers, and neither was usable here. Tempo's was off, and the config the image hands out assumes Grafana's default port, which this stack deliberately moves. Following the shipped copy connects to nothing, and the failure is quiet in the worst way: the agent authenticates against nothing, finds no data, and reports an empty stack rather than a misconfigured one. what: - Enable Tempo's MCP server, which offers traceql-search, get-trace, and attribute discovery, so an agent can find the engine attributes without being told the schema; verified by driving the handshake and searching for this repo's own spans - Add just otel-mcp, printing a client config with this stack's real ports and the running container's token, which is never written to the repository - Test that the config and the stack agree on the Grafana port, since drift there fails silently --- justfile | 5 ++++ scripts/lgtm/README.md | 24 ++++++++++++++++++ scripts/lgtm/mcp-config.sh | 48 +++++++++++++++++++++++++++++++++++ scripts/lgtm/up.sh | 6 ++++- tests/test_lgtm_dashboards.py | 26 +++++++++++++++++++ 5 files changed, 108 insertions(+), 1 deletion(-) create mode 100755 scripts/lgtm/mcp-config.sh diff --git a/justfile b/justfile index 19a269471..be0abd78d 100644 --- a/justfile +++ b/justfile @@ -160,6 +160,11 @@ otel-down: otel-ports: scripts/lgtm/verify.sh +# Print an MCP client config for this stack, with its real ports and token +[group: 'otel'] +otel-mcp: + scripts/lgtm/mcp-config.sh + # Regenerate the provisioned Grafana dashboards from their generator [group: 'otel'] otel-dashboards: diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md index 31a6ddfcc..94600f179 100644 --- a/scripts/lgtm/README.md +++ b/scripts/lgtm/README.md @@ -244,6 +244,30 @@ check therefore re-queries the panels that came back empty until they fill or only as long as it needs. A cold-started Tempo is the usual reason for a second pass. +## Querying it from an agent + +Both Grafana and Tempo expose MCP servers, so an agent can ask the stack +questions directly instead of being handed screenshots. Print a client +configuration for this stack: + +```console +$ just otel-mcp +``` + +Use its output rather than the one at `/etc/lgtm/mcp.json` inside the +container. The shipped config assumes Grafana is on its default port; this +stack moves it, so following the shipped copy connects to nothing and an agent +reports an empty stack rather than a misconfigured one. + +The Grafana service account token is read from the running container each time +and is never stored in this repository. + +Tempo's MCP server answers at `http://127.0.0.1:3200/api/mcp` and offers seven +tools, including `traceql-search`, `get-trace`, and `get-attribute-values`, so +the engine attributes are discoverable without knowing the schema in advance. +Asking it for `{ resource.service.name="libtmux-engines" }` returns the same +spans the dashboards draw. + ## Configuration `up.sh` pins the `grafana/otel-lgtm` image rather than tracking `latest`, so a diff --git a/scripts/lgtm/mcp-config.sh b/scripts/lgtm/mcp-config.sh new file mode 100755 index 000000000..cdf725c2a --- /dev/null +++ b/scripts/lgtm/mcp-config.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Print an MCP client configuration for this stack. +# +# The image ships its own at /etc/lgtm/mcp.json, but it assumes the default +# ports. This stack moves Grafana off 3000 so it does not collide with whatever +# else is running, which makes the shipped config point somewhere with nothing +# behind it -- an agent would connect, get no data, and report that the stack is +# empty rather than that it is misconfigured. +# +# The Grafana service account token is read from the running container each +# time. It is never written into the repository. +set -euo pipefail + +CONTAINER="${LIBTMUX_LGTM_CONTAINER:-libtmux-lgtm}" +GRAFANA_PORT="${LIBTMUX_LGTM_GRAFANA_PORT:-3900}" + +if ! docker inspect "$CONTAINER" > /dev/null 2>&1; then + echo "the stack is not running; start it with: just otel-up" >&2 + exit 1 +fi + +token="$( + docker exec "$CONTAINER" cat /etc/lgtm/mcp.json 2> /dev/null \ + | sed -n 's/.*"GRAFANA_SERVICE_ACCOUNT_TOKEN": *"\([^"]*\)".*/\1/p' +)" +if [[ -z "$token" ]]; then + echo "no Grafana service account token found in $CONTAINER" >&2 + exit 1 +fi + +cat < None: creation = source.index("def _server(") assert validation < creation, "the lane must be validated before _server is defined" assert "raise SystemExit(message)" in source + + +def test_the_mcp_config_and_the_stack_agree_on_grafana_port() -> None: + """An agent's config must point where the stack actually puts Grafana. + + The image ships an MCP config assuming Grafana's default port. This stack + moves it, so the shipped copy connects to nothing -- and the failure is + quiet in the worst way: the agent authenticates against nothing, finds no + data, and reports an empty stack rather than a misconfigured one. Keeping + the two defaults in step is the whole point of shipping our own. + """ + import re + + up = (_LGTM / "up.sh").read_text(encoding="utf-8") + mcp = (_LGTM / "mcp-config.sh").read_text(encoding="utf-8") + + def default_port(text: str) -> str: + match = re.search( + r'GRAFANA_PORT="\$\{LIBTMUX_LGTM_GRAFANA_PORT:-(\d+)\}"', text + ) + assert match, "no Grafana port default found" + return match.group(1) + + assert default_port(up) == default_port(mcp) + # The token belongs to the running container, never to the repository. + assert "glsa_" not in mcp From b07e0e043a19d9453f051c40bc20ccfb8c38e1fd Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 14:44:14 -0500 Subject: [PATCH 13/24] Bench(feat[lgtm]): Add allocation profiles behind a flag why: Only CPU sampling was ever collected, so "where did the time go" had an answer and "what did it allocate" did not. Pyroscope lists the other profile types it knows, which made the gap look like a wiring fault rather than a choice. what: - Add --memory-profile to the smoke workload, collecting alloc_space, alloc_objects, and inuse_space; left off by default because allocation profiling costs more than sampling - Record that these are per run rather than per transport: the lane tag scopes the CPU sampler and the allocation profiler does not consult it, so claiming otherwise would be wrong - Record that the goroutine, mutex, and block types Pyroscope advertises stay empty for a Python process, so an empty panel there is expected --- scripts/lgtm/README.md | 16 +++++++++++++++- scripts/otel_smoke.py | 9 +++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md index 94600f179..ab8ff3e38 100644 --- a/scripts/lgtm/README.md +++ b/scripts/lgtm/README.md @@ -97,7 +97,21 @@ Logs carry trace context, so a log line links to the trace it came from. Profiles come from Pyroscope sampling the process, which is how "where did the Python time go" gets answered — the engines' own frames show up in the flame -graph. +graph, tagged per transport, so a lane's CPU cost is as comparable as its +latency. + +Allocation profiles are available too, but off by default because collecting +them costs more than CPU sampling: + +```console +$ just otel-smoke --memory-profile +``` + +That adds the `memory:alloc_space`, `memory:alloc_objects`, and +`memory:inuse_space` profile types. They are per run rather than per transport: +the per-lane tag scopes the CPU sampler, and the allocation profiler does not +consult it. The other profile types Pyroscope lists — goroutines, mutex, block +— belong to Go runtimes and stay empty for a Python process. The workload deliberately issues commands tmux rejects. A dashboard whose error panel is empty is untested rather than healthy, so the failure path has to diff --git a/scripts/otel_smoke.py b/scripts/otel_smoke.py index e358e4fdb..4000875f7 100644 --- a/scripts/otel_smoke.py +++ b/scripts/otel_smoke.py @@ -211,6 +211,11 @@ def main(argv: list[str] | None = None) -> int: "--pyroscope", default=os.environ.get("PYROSCOPE_SERVER_ADDRESS", "http://127.0.0.1:4040"), ) + parser.add_argument( + "--memory-profile", + action="store_true", + help="also collect allocation profiles (costs more than CPU sampling)", + ) args = parser.parse_args(argv) os.environ.pop("TMUX", None) @@ -227,6 +232,10 @@ def main(argv: list[str] | None = None) -> int: server_address=args.pyroscope, sample_rate=100, upload_interval=3, + # Allocation profiling answers a different question from CPU + # sampling and costs more to collect, so it stays opt-in rather + # than being switched on for every run. + mem_enabled=args.memory_profile, # The full static identity: a profile is one process, so branch, # revision, worktree, and spike cost nothing extra here and make # two runs directly comparable in Pyroscope. From 380d6353bec31789d2bc680f823433b299563502 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 16:55:28 -0500 Subject: [PATCH 14/24] Bench(fix[lgtm]): Name the remedy when a port is shadowed why: The check reported that a published port reaches something other than this container, which is the hard half of the problem, and then stopped. The reader is left knowing they have a collision and not that the port is overridable, so the obvious next move is to start deleting containers. what: - Print the environment variable that republishes the affected service, and only for the two this stack actually moves; naming one for a service that keeps its upstream default would send the reader somewhere useless - Have up.sh show the same remedy when it refuses to report success --- scripts/lgtm/up.sh | 4 +++- scripts/lgtm/verify.sh | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/lgtm/up.sh b/scripts/lgtm/up.sh index 70c3b33d2..cdf9e85bf 100755 --- a/scripts/lgtm/up.sh +++ b/scripts/lgtm/up.sh @@ -94,7 +94,9 @@ done # Identity, not liveness: a port that answers may belong to a host process # that was already bound to it. verify.sh tells the two apart. "$ROOT/scripts/lgtm/verify.sh" || { - echo "a published port does not reach this container; see the lines above" >&2 + echo "a published port does not reach this container; see the lines above," >&2 + echo "then republish on a free port, for example:" >&2 + echo " LIBTMUX_LGTM_PROM_PORT=9098 just otel-up" >&2 exit 1 } diff --git a/scripts/lgtm/verify.sh b/scripts/lgtm/verify.sh index d40c1e88c..03fadc975 100755 --- a/scripts/lgtm/verify.sh +++ b/scripts/lgtm/verify.sh @@ -17,6 +17,16 @@ GRAFANA_PORT="${LIBTMUX_LGTM_GRAFANA_PORT:-3900}" PROM_PORT="${LIBTMUX_LGTM_PROM_PORT:-9099}" status=0 +port_variable() { + # Only the two services this stack moves are overridable; the rest keep + # their upstream defaults, so naming a variable for them would mislead. + case "$1" in + grafana) printf 'LIBTMUX_LGTM_GRAFANA_PORT' ;; + prometheus) printf 'LIBTMUX_LGTM_PROM_PORT' ;; + *) printf 'LIBTMUX_LGTM_CONTAINER' ;; + esac +} + check() { local label=$1 inside_port=$2 host_port=$3 path=$4 local inside outside @@ -29,6 +39,8 @@ check() { printf ' %-11s %-6s ok\n' "$label" "$host_port" else printf ' %-11s %-6s SHADOWED by another service on this port\n' "$label" "$host_port" + printf ' %-11s %-6s publish it elsewhere: %s= just otel-up\n' \ + "" "" "$(port_variable "$label")" status=1 fi } From 0e8e1c47a8e4bb084f3829adab15789c85f71ba1 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 18:01:23 -0500 Subject: [PATCH 15/24] Bench(feat[lgtm]): Measure control-mode streaming why: Notifications arrive out of band, so the instrumentation seam never sees them -- a sink wraps run(), and nothing routes a %output through run(). That left the streaming half of control mode unmeasured, including the engine's own count of notifications dropped when a subscriber falls behind, which is the number that says the stream is unhealthy. what: - Subscribe to notifications while commands keep flowing, recording those received and those dropped; 240 arrive in a three second lane with none dropped, and the commands are unaffected - Establish the subscription before generating output and pace the sends, because a tight command loop starves the consumer and it then waits for output that has already gone by - Give the lane its own shell window: the workload's panes run sleep, which ignores keystrokes and emits nothing - Read the counters with max_over_time rather than increase, since a value written once per run is flat and increase over it is zero --- scripts/lgtm/README.md | 8 + scripts/lgtm/dashboards/libtmux-overview.json | 148 +++++++++++++++++- scripts/lgtm/generate_dashboards.py | 38 +++++ scripts/otel_smoke.py | 70 +++++++++ 4 files changed, 257 insertions(+), 7 deletions(-) diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md index ab8ff3e38..4b583978b 100644 --- a/scripts/lgtm/README.md +++ b/scripts/lgtm/README.md @@ -95,6 +95,14 @@ metric-to-trace pivot. Logs carry trace context, so a log line links to the trace it came from. +Streaming is measured separately, because notifications arrive out of band and +the instrumentation seam never sees them: a sink wraps `run()`, and nothing +routes a `%output` through `run()`. The workload therefore subscribes to +control-mode notifications while commands keep flowing, and records how many +arrived along with the engine's own count of any it had to drop because the +subscriber fell behind. Those two counters are written once per run, so their +panels read the last value in the window rather than an increase across it. + Profiles come from Pyroscope sampling the process, which is how "where did the Python time go" gets answered — the engines' own frames show up in the flame graph, tagged per transport, so a lane's CPU cost is as comparable as its diff --git a/scripts/lgtm/dashboards/libtmux-overview.json b/scripts/lgtm/dashboards/libtmux-overview.json index 006b50374..58a295517 100644 --- a/scripts/lgtm/dashboards/libtmux-overview.json +++ b/scripts/lgtm/dashboards/libtmux-overview.json @@ -613,7 +613,7 @@ }, { "type": "row", - "title": "Traces, logs, and profiles", + "title": "Streaming", "collapsed": false, "id": 13, "gridPos": { @@ -624,6 +624,140 @@ }, "panels": [] }, + { + "type": "stat", + "title": "Notifications received", + "description": "Control-mode notifications consumed by a subscriber. Counted once per run, so this reads the last value in the window rather than an increase over it.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max_over_time(sum(tmux_notifications_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"})[$__range:])", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "id": 14, + "gridPos": { + "x": 0, + "y": 34, + "w": 12, + "h": 5 + } + }, + { + "type": "stat", + "title": "Notifications dropped", + "description": "Notifications discarded because a subscriber fell behind. Above zero means the stream is outrunning its consumer.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max_over_time(sum(tmux_notifications_dropped_total{tmux_lane=~\"$lane\", vcs_ref_head_name=~\"$branch\"})[$__range:])", + "legendFormat": "__auto", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "id": 15, + "gridPos": { + "x": 12, + "y": 34, + "w": 12, + "h": 5 + } + }, + { + "type": "row", + "title": "Traces, logs, and profiles", + "collapsed": false, + "id": 16, + "gridPos": { + "x": 0, + "y": 39, + "w": 24, + "h": 1 + }, + "panels": [] + }, { "type": "table", "title": "Requests that batched commands", @@ -648,10 +782,10 @@ "options": { "showHeader": true }, - "id": 14, + "id": 17, "gridPos": { "x": 0, - "y": 34, + "y": 40, "w": 12, "h": 8 } @@ -684,10 +818,10 @@ "wrapLogMessage": true, "enableLogDetails": true }, - "id": 15, + "id": 18, "gridPos": { "x": 12, - "y": 34, + "y": 40, "w": 12, "h": 8 } @@ -714,10 +848,10 @@ } ], "options": {}, - "id": 16, + "id": 19, "gridPos": { "x": 0, - "y": 42, + "y": 48, "w": 24, "h": 11 } diff --git a/scripts/lgtm/generate_dashboards.py b/scripts/lgtm/generate_dashboards.py index c6bf20cc0..dc43d4ea4 100644 --- a/scripts/lgtm/generate_dashboards.py +++ b/scripts/lgtm/generate_dashboards.py @@ -640,6 +640,44 @@ def build_overview() -> Board: links=[drill("Break down by command", "libtmux-commands")], ) + board.row("Streaming") + board.add( + stat( + "Notifications received", + [ + target( + f"max_over_time(sum(tmux_notifications_total{{{SCOPE}}})[$__range:])", + instant=True, + ) + ], + description=( + "Control-mode notifications consumed by a subscriber. Counted " + "once per run, so this reads the last value in the window " + "rather than an increase over it." + ), + ), + w=12, + h=5, + ) + board.add( + stat( + "Notifications dropped", + [ + target( + f"max_over_time(sum(tmux_notifications_dropped_total{{{SCOPE}}})[$__range:])", + instant=True, + ) + ], + thresholds=ERR_THRESHOLDS, + description=( + "Notifications discarded because a subscriber fell behind. " + "Above zero means the stream is outrunning its consumer." + ), + ), + w=12, + h=5, + ) + board.row("Traces, logs, and profiles") board.add( traces( diff --git a/scripts/otel_smoke.py b/scripts/otel_smoke.py index 4000875f7..2fce76ce4 100644 --- a/scripts/otel_smoke.py +++ b/scripts/otel_smoke.py @@ -160,6 +160,72 @@ def _profile_lane(lane: str, *, enabled: bool) -> t.Iterator[None]: yield +async def stream_lane(server: t.Any, signals: t.Any, seconds: float) -> dict[str, int]: + """Consume control-mode notifications while commands keep flowing. + + Notifications arrive out of band, so the instrumentation seam never sees + them: a sink wraps run(), and nothing routes a %output through run(). That + leaves the streaming half of control mode unmeasured, including the + engine's own count of notifications it had to drop because a subscriber + fell behind -- which is the number that says the stream is unhealthy. + + Reading the counter costs one attribute access at the end of the lane, so + the measurement does not disturb what it measures. + """ + engine = AsyncControlModeEngine.for_server(server) + # The workload's own panes run `sleep`, which ignores keystrokes and so + # emits no output. Streaming needs something that answers, so this lane + # gets its own shell window; without one the notification count is a + # confident zero. + await engine.run( + CommandRequest.from_args("new-window", "-t", "smoke", "-n", "stream", "sh") + ) + received = 0 + deadline = time.monotonic() + seconds + + async def consume() -> None: + nonlocal received + async for _ in engine.subscribe(): + received += 1 + if time.monotonic() > deadline: + break + + consumer = asyncio.create_task(consume()) + # Let the consumer register its subscription before any output exists. + # Started against a tight command loop it never gets scheduled, then waits + # for notifications the shell already emitted, and reports a confident + # zero. + await asyncio.sleep(0.1) + try: + while time.monotonic() < deadline: + await engine.run( + CommandRequest.from_args( + "send-keys", "-t", "stream", "echo stream", "Enter" + ) + ) + # Pace the sends so producer and consumer interleave. Saturating + # the connection would measure a queue, not a stream. + await asyncio.sleep(0.02) + try: + await asyncio.wait_for(consumer, timeout=5) + except TimeoutError: + consumer.cancel() + dropped = engine.dropped_notifications + finally: + await engine.aclose() + + meter = signals.meter + labels = {**signals.metric_labels, "tmux.lane": "control-stream"} + meter.create_counter( + "tmux.notifications", description="Control-mode notifications received." + ).add(received, labels) + meter.create_counter( + "tmux.notifications.dropped", + description="Notifications dropped because a subscriber fell behind.", + ).add(dropped, labels) + return {"received": received, "dropped": dropped} + + def _log_lane(signals: t.Any, lane: str, totals: dict[str, int]) -> None: """Log a lane's result from inside a span, so the line links to a trace. @@ -304,6 +370,10 @@ async def drive(engine: t.Any = engine) -> None: asyncio.run(drive()) totals[lane] = lane_totals(counts) _log_lane(signals, lane, totals[lane]) + + with telemetry.scope(**{"libtmux.phase": "control-stream"}): + stream = asyncio.run(stream_lane(server, signals, args.seconds)) + logger.info("stream finished", extra={"lane": "control-stream", **stream}) finally: subprocess.run( ("tmux", "-S", str(root / "smoke.sock"), "kill-server"), From 0ec50fb102d182d32890a728b6934fe2c5c039f6 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 18:14:10 -0500 Subject: [PATCH 16/24] Bench(docs[lgtm]): Record which tmux builds the stack was checked against why: The telemetry work had only ever run against one tmux, and control mode is the part most likely to drift between releases -- its notification set and client flags have both changed over the versions libtmux supports. A build that emitted nothing would leave the streaming panels blank with no clue why. Exercised on 3.2a, 3.3a, 3.4, 3.5, 3.6, 3.7, 3.7a and 3.7b: identical command and inlining counts on both transports, notifications delivered on every one, none dropped. what: - State the range in the README, since "works with tmux" is not a claim a reader can act on --- scripts/lgtm/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md index 4b583978b..f3f510766 100644 --- a/scripts/lgtm/README.md +++ b/scripts/lgtm/README.md @@ -121,6 +121,11 @@ the per-lane tag scopes the CPU sampler, and the allocation profiler does not consult it. The other profile types Pyroscope lists — goroutines, mutex, block — belong to Go runtimes and stay empty for a Python process. +Both transports and the notification stream were exercised against every tmux +from 3.2a to 3.7b, with identical command and inlining counts and no dropped +notifications on any of them. Control mode is the part most likely to drift +between releases, so that is the half worth having checked. + The workload deliberately issues commands tmux rejects. A dashboard whose error panel is empty is untested rather than healthy, so the failure path has to produce real data. From b9d53887371e7119d41670fb95d34cd02ebd126a Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Thu, 20 Aug 2026 18:41:36 -0500 Subject: [PATCH 17/24] Bench(fix[lgtm]): Wait for a port instead of sampling it once why: A cold start failed with grafana UNREACHABLE, and the same check passed seconds later. The container reports healthy before every service inside it has finished binding, so verifying immediately catches Grafana mid-startup and refuses a stack that is fine. Shadowing is deliberately not retried: two different services answering the same port is a settled fact, not a timing question, and retrying it would only delay a report that will not change. what: - Retry a port that is not answering yet, up to LIBTMUX_LGTM_WAIT seconds; two consecutive cold starts now pass where they previously failed, and a healthy stack still verifies in under two seconds - Record what has been verified in scripts/lgtm/VERIFICATION.md: transport against signal, tmux 3.2a through 3.7b, load shapes, profile types, identity resolution, failure paths, and what is deliberately not covered - Recompute the panel total from the boards in a test, so the record cannot claim a number the dashboards do not define --- scripts/lgtm/README.md | 3 +- scripts/lgtm/VERIFICATION.md | 113 ++++++++++++++++++++++++++++++++++ scripts/lgtm/verify.sh | 19 +++++- tests/test_lgtm_dashboards.py | 33 +++++++++- 4 files changed, 161 insertions(+), 7 deletions(-) create mode 100644 scripts/lgtm/VERIFICATION.md diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md index f3f510766..ae1bb0ef0 100644 --- a/scripts/lgtm/README.md +++ b/scripts/lgtm/README.md @@ -21,7 +21,8 @@ $ just otel-verify ``` That is the command to reach for first. The steps below are the same workflow -taken one piece at a time. +taken one piece at a time. What that command covers, and what it deliberately +does not, is recorded in [VERIFICATION.md](VERIFICATION.md). Start or restart the stack: diff --git a/scripts/lgtm/VERIFICATION.md b/scripts/lgtm/VERIFICATION.md new file mode 100644 index 000000000..848e82ee2 --- /dev/null +++ b/scripts/lgtm/VERIFICATION.md @@ -0,0 +1,113 @@ +# What has been verified, and how + +This records what was exercised and by which command, so a claim here can be +re-run rather than taken on trust. Every row was executed against a live stack +and a real tmux server. + +Reproduce the whole thing with one command: + +```console +$ just otel-verify +``` + +## Permutation matrix + +### Transport against signal + +One smoke run drives all four transports; each cell was queried from its own +backend afterwards. **16/16 populated.** + +| transport | metrics | traces | logs | profiles | +| --------- | ------- | ------ | ---- | -------- | +| `subprocess` | yes | yes | yes | yes | +| `control` | yes | yes | yes | yes | +| `subprocess-async` | yes | yes | yes | yes | +| `control-async` | yes | yes | yes | yes | + +Profiles are per transport because the CPU sampler is tagged per lane. +Allocation profiles are per run; see below. + +### tmux versions + +Both transports plus the notification stream, against every build. Identical +command and inlining counts on all of them, no dropped notifications. +**8/8 exercised.** + +| 3.2a | 3.3a | 3.4 | 3.5 | 3.6 | 3.7 | 3.7a | 3.7b | +| ---- | ---- | --- | --- | --- | --- | ---- | ---- | +| yes | yes | yes | yes | yes | yes | yes | yes | + +### Load shapes + +`just otel-load`, two transports against two executors. **4/4, no check +failures.** + +| | `steady` (constant VUs) | `ramp` (ramping arrival rate) | +| --- | --- | --- | +| `control-async` | yes | yes | +| `subprocess-async` | yes | yes | + +### Profile types + +| type | collected | scope | +| ---- | --------- | ----- | +| `process_cpu:cpu:nanoseconds` | always | per transport | +| `memory:alloc_space:bytes` | `--memory-profile` | per run | +| `memory:alloc_objects:count` | `--memory-profile` | per run | +| `memory:inuse_space:bytes` | `--memory-profile` | per run | + +The goroutine, mutex, and block types Pyroscope advertises belong to Go +runtimes and stay empty for a Python process. + +### Identity resolution + +| case | result | +| ---- | ------ | +| branch checkout | branch name, type `branch` | +| detached HEAD | short revision, type `revision`, never the literal `HEAD` | +| detached at a tag | tag name, type `tag` | +| `LIBTMUX_VCS_REF` set | the override wins over the checkout | +| outside a repository | no `vcs.*` attributes, no error | + +### Failure paths + +Each fails loudly and names the next action. + +| case | behaviour | +| ---- | --------- | +| stack not running | acceptance exits in ~18s, prints `just otel-up` | +| port shadowed by a host service | `up.sh` refuses, prints the override variable | +| dashboards edited by hand | test fails, prints `just otel-dashboards` | +| unknown load transport | exits in ~3s, lists the valid ones | + +## Dimension scorecard + +Each rating is backed by something executed, not an opinion. + +| dimension | evidence | +| --------- | -------- | +| Dashboards | 43/43 panel queries return data, checked against the panels' own JSON | +| Examples | 64 doctested examples; every console block in this directory was run as written | +| Docs | tests fail if the README shows a recipe that does not exist, or omits a generated board | +| Scannability | one command to a verified stack; 9/9 tasks carry descriptions | +| MCP | Tempo's 7 tools reached over a real handshake; `just otel-mcp` emits a config with this stack's ports, token never committed | +| async | concurrent scopes attribute correctly; overlapping tasks preserved under load | +| control mode | both transports on 8 tmux builds | +| streaming | 240 notifications consumed in a 3s lane, none dropped, commands unaffected | +| asyncio correctness | full run under `PYTHONASYNCIODEBUG=1`: no slow callbacks, un-awaited coroutines, or un-retrieved task exceptions | +| benchmarking | open-model load finds saturation the fixed-worker loop hides: p99 about 2ms steady against about 16ms at the top of a ramp | +| profiling | CPU profile filterable per transport; a bogus transport returns nothing, so the filter is real | + +## Deliberately not covered + +Allocation profiles are not per transport. The lane tag scopes the CPU +sampler and the allocation profiler does not consult it, so claiming otherwise +would be wrong. + +Streaming counters are per-run scalars rather than per-notification, so they +answer "did this run drop anything" and not "when". A rate would cost a metric +record per notification, which is real overhead on a hot stream. + +Cross-repository telemetry has nothing to permute yet: the other libtmux ports +have no instrumentation seam, so adding one is separate work rather than a +test. diff --git a/scripts/lgtm/verify.sh b/scripts/lgtm/verify.sh index 03fadc975..8d9d70f5b 100755 --- a/scripts/lgtm/verify.sh +++ b/scripts/lgtm/verify.sh @@ -27,11 +27,24 @@ port_variable() { esac } +# Seconds to keep waiting for a port that is not answering yet. A service can +# still be binding when its container reports healthy, so a single sample +# turns a cold start into a spurious failure. Shadowing is not retried: two +# different services answering is a settled fact, not a timing question. +WAIT_SECONDS="${LIBTMUX_LGTM_WAIT:-30}" + check() { local label=$1 inside_port=$2 host_port=$3 path=$4 - local inside outside - inside=$(docker exec "$CONTAINER" curl -s -m 8 "http://127.0.0.1:${inside_port}${path}" 2> /dev/null) - outside=$(curl -s -m 8 "http://127.0.0.1:${host_port}${path}" 2> /dev/null) + local inside outside deadline + deadline=$((SECONDS + WAIT_SECONDS)) + while :; do + inside=$(docker exec "$CONTAINER" curl -s -m 8 "http://127.0.0.1:${inside_port}${path}" 2> /dev/null) + outside=$(curl -s -m 8 "http://127.0.0.1:${host_port}${path}" 2> /dev/null) + if [[ -n "$outside" && -n "$inside" ]] || ((SECONDS >= deadline)); then + break + fi + sleep 2 + done if [[ -z "$outside" ]]; then printf ' %-11s %-6s UNREACHABLE from host\n' "$label" "$host_port" status=1 diff --git a/tests/test_lgtm_dashboards.py b/tests/test_lgtm_dashboards.py index 3311e6145..b6718d1e2 100644 --- a/tests/test_lgtm_dashboards.py +++ b/tests/test_lgtm_dashboards.py @@ -299,16 +299,43 @@ def test_readme_only_shows_commands_that_exist() -> None: """ import re - readme = (_LGTM / "README.md").read_text(encoding="utf-8") + prose = "\n".join( + (_LGTM / name).read_text(encoding="utf-8") + for name in ("README.md", "VERIFICATION.md") + ) justfile = (_ROOT / "justfile").read_text(encoding="utf-8") - shown = set(re.findall(r"^\$ just ([a-z][a-z-]*)", readme, re.MULTILINE)) + shown = set(re.findall(r"^\$ just ([a-z][a-z-]*)", prose, re.MULTILINE)) assert shown, "the README stopped showing any just commands" defined = set( re.findall(r"^([a-z][a-z-]*)(?: \*?[a-z_]+)?:", justfile, re.MULTILINE) ) missing = shown - defined - assert not missing, f"README shows recipes that do not exist: {sorted(missing)}" + assert not missing, f"the docs show recipes that do not exist: {sorted(missing)}" + + +def test_the_verification_record_counts_the_panels_it_claims() -> None: + """The recorded panel total must match the boards actually generated. + + A verification document that drifts is worse than none: it reads as + evidence while describing something else. The panel count is the number a + reader is most likely to trust without re-running anything. + """ + import re + + record = (_LGTM / "VERIFICATION.md").read_text(encoding="utf-8") + claimed = re.search(r"(\d+)/(\d+) panel queries", record) + assert claimed, "the verification record no longer states a panel total" + + actual = 0 + for path in sorted(_DASHBOARDS.glob("*.json")): + board = json.loads(path.read_text(encoding="utf-8")) + for panel in _query_panels(board): + actual += len(panel["targets"]) + assert int(claimed.group(2)) == actual, ( + f"the record claims {claimed.group(2)} panel queries; " + f"the boards define {actual}" + ) def test_readme_names_every_dashboard_it_ships() -> None: From 30439c2a487aae377e778d4be72264cbfc835297 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Fri, 21 Aug 2026 12:33:09 -0500 Subject: [PATCH 18/24] Bench(fix[lgtm]): Reclaim what a killed load run strands why: Cleanup lived only in rampa's teardown hook. SIGINT reaches it, but SIGKILL runs no user code, so a killed run stranded its tmux server, that server's pane, and its scratch root, each holding a pty, with nothing left to reclaim them. Repeated interrupted runs walk toward pty exhaustion. Separately, the pane's holding command is a run-duration ceiling rather than a cleanup mechanism: when it exits the window closes, the last window closing ends the session, and the server goes with it. destroy-unattached off only survives detach. At sleep 600 any run past ten minutes lost its server mid-flight. what: - Reap roots whose owner is proven absent, at the start of the next run -- the one path that survives an exit running no user code - Stamp each root with its owner's pid and start time, so a reused pid cannot be mistaken for a live run - Leave a root alone while its owner runs, and leave a pre-owner-file root alone while a tmux still answers on its socket - Raise the holding command above any plausible --duration - Cover both directions, including the reused-pid case --- scripts/lgtm/load_tmux.py | 107 +++++++++++++++++++++- tests/test_lgtm_load.py | 186 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 291 insertions(+), 2 deletions(-) create mode 100644 tests/test_lgtm_load.py diff --git a/scripts/lgtm/load_tmux.py b/scripts/lgtm/load_tmux.py index a0846b538..5e9a06986 100644 --- a/scripts/lgtm/load_tmux.py +++ b/scripts/lgtm/load_tmux.py @@ -16,11 +16,13 @@ from __future__ import annotations import asyncio +import contextlib import os import pathlib import shutil import subprocess import sys +import tempfile import typing as t import uuid @@ -75,11 +77,108 @@ # transport, which is the opposite of the point. _STATE: dict[str, t.Any] = {} +_ROOT_PREFIX = "libtmux-load-" + +# The holding command for the server's one pane. When it exits the window +# closes, the last window closing ends the session, and the server exits with +# it -- `destroy-unattached off` does not prevent that, it only survives +# *detach*. So this value is the ceiling on how long a run can last, not a +# cleanup mechanism, and it is set far above any plausible `--duration`. +# Cleanup is `teardown` plus `_reap_stale_roots`, which do not depend on it. +_HOLD_COMMAND = "sleep 86400" + + +def _process_identity(pid: int) -> str | None: + """Return a pid's start time, or ``None`` when the pid is not running. + + Pairing the pid with its start time is what makes a staleness claim safe: + pids are reused, and reaping on a bare pid check could delete a live run's + server the moment the kernel handed its number to something else. + """ + try: + stat = pathlib.Path(f"/proc/{pid}/stat").read_bytes() + except OSError: + return None + # `comm` is parenthesised and may itself contain spaces or parens, so the + # fields are counted from the last ')' rather than by splitting the line. + fields = stat[stat.rfind(b")") + 2 :].split() + try: + return fields[19].decode() + except IndexError: + return None + + +def _reap_stale_roots() -> None: + """Remove load roots whose owning process is gone. + + ``teardown`` is the normal cleanup path and rampa runs it after SIGINT too. + SIGKILL runs no user code at all, so a killed run strands its tmux server, + that server's pane, and this directory -- each holding a pty -- with nothing + left to reclaim them. This is the path that survives any exit, because it + runs at the *start* of the next run rather than the end of the last one. + + A root is only removed once its owner is proven absent. A root whose owner + is still running belongs to a concurrent run and is left alone; so is a root + with no owner file that still has a tmux bound to it, since stealing another + run's server would be worse than leaking this one. + """ + reaped = 0 + for root in pathlib.Path(tempfile.gettempdir()).glob(f"{_ROOT_PREFIX}*"): + if not root.is_dir(): + continue + socket_path = root / "load.sock" + owner = root / "owner" + if owner.exists(): + try: + pid_text, identity = owner.read_text().split() + if _process_identity(int(pid_text)) == identity: + continue # a live run owns this root + except (OSError, ValueError): + pass # unreadable owner file: fall through to the socket probe + elif _socket_has_server(socket_path): + continue # predates the owner file and is still in use + with contextlib.suppress(Exception): + subprocess.run( + ("tmux", "-S", str(socket_path), "kill-server"), + capture_output=True, + check=False, + ) + shutil.rmtree(root, ignore_errors=True) + reaped += 1 + if reaped: + print(f"reaped {reaped} stale load root(s)", file=sys.stderr) + + +def _socket_has_server(socket_path: pathlib.Path) -> bool: + """Report whether a tmux server is answering on *socket_path*. + + ``kill-server`` and every other tmux subcommand exit 1 when no server is + listening, so the check is the exit status of the cheapest read-only + command rather than the presence of the socket file, which outlives its + server. + """ + try: + return ( + subprocess.run( + ("tmux", "-S", str(socket_path), "list-sessions"), + capture_output=True, + check=False, + timeout=5, + ).returncode + == 0 + ) + except (OSError, subprocess.SubprocessError): + return False + def _server() -> tuple[Server, pathlib.Path]: """Create the throwaway tmux server this run drives.""" - root = pathlib.Path(f"/tmp/libtmux-load-{uuid.uuid4().hex[:8]}") + root = pathlib.Path(tempfile.gettempdir()) / f"{_ROOT_PREFIX}{uuid.uuid4().hex[:8]}" root.mkdir(mode=0o700) + # Written before the server exists, so a kill between mkdir and new-session + # still leaves a root the next run can prove stale. + identity = _process_identity(os.getpid()) + (root / "owner").write_text(f"{os.getpid()} {identity}") socket_path = root / "load.sock" subprocess.run( ( @@ -92,7 +191,7 @@ def _server() -> tuple[Server, pathlib.Path]: "-d", "-s", "load", - "sleep 600", + _HOLD_COMMAND, ), check=True, ) @@ -137,6 +236,10 @@ def _engine() -> t.Any: # Resolve the factory before anything is created, so a failure here # cannot leave a tmux server behind. factory = LANES[LANE] + # Reclaim what an earlier killed run stranded. Runs here rather than + # at import so a scenario listing does not touch other runs' roots, + # and behind the latch so it happens once per run, not per iteration. + _reap_stale_roots() server, root = _server() _STATE["root"] = root signals = telemetry.build( diff --git a/tests/test_lgtm_load.py b/tests/test_lgtm_load.py new file mode 100644 index 000000000..5e95dbb02 --- /dev/null +++ b/tests/test_lgtm_load.py @@ -0,0 +1,186 @@ +"""Behavioral checks for the rampa load scenario's stale-root reaper. + +``scripts/lgtm/load_tmux.py`` cleans up in rampa's ``teardown`` hook, which +covers a normal finish and a Ctrl-C alike. SIGKILL runs no user code, so a +killed run strands a tmux server, its pane, and its scratch root, each holding +a pty. The reaper is the path that survives that, because it runs at the start +of the *next* run. + +These run the scenario module in a subprocess under the ``load`` dependency +group: rampa needs a newer Python than libtmux targets, so it is not present in +the plain test environment. +""" + +from __future__ import annotations + +import os +import pathlib +import subprocess +import textwrap + +import pytest + +_ROOT = pathlib.Path(__file__).parents[1] + + +_PREAMBLE = ( + "import os, pathlib, sys\nsys.path.insert(0, 'scripts/lgtm')\nimport load_tmux\n" +) + + +def _run_in_load_env(body: str) -> subprocess.CompletedProcess[str]: + """Execute *body* with the scenario module already imported. + + *body* is dedented on its own before the preamble is prepended: dedenting + the joined text instead would find no common prefix and leave the indented + half indented, which fails as an ``IndentationError`` the caller then reads + as "rampa is unavailable" and skips. + """ + env = os.environ.copy() + env.pop("VIRTUAL_ENV", None) + # The scenario refuses to import without a valid lane, by design. + env["LIBTMUX_LOAD_LANE"] = "control-async" + return subprocess.run( + ( + "uv", + "run", + "--group", + "otel", + "--group", + "load", + "python", + "-c", + _PREAMBLE + textwrap.dedent(body), + ), + cwd=_ROOT, + env=env, + text=True, + capture_output=True, + check=False, + ) + + +@pytest.fixture(scope="module") +def load_module_available() -> None: + """Skip the module when rampa cannot be resolved for this interpreter.""" + probe = _run_in_load_env("print('importable')") + if probe.returncode != 0: + pytest.skip(f"load scenario not importable here: {probe.stderr[-400:]}") + assert "importable" in probe.stdout + + +def test_reaper_removes_a_root_whose_owner_is_gone( + load_module_available: None, + tmp_path: pathlib.Path, +) -> None: + """A root left by a killed run is reclaimed, server and directory alike.""" + completed = _run_in_load_env( + f""" + root = pathlib.Path({str(tmp_path)!r}) / (load_tmux._ROOT_PREFIX + "dead") + root.mkdir() + # pid 2**22 is above the default pid_max, so it cannot be running. + (root / "owner").write_text("4194304 999") + + load_tmux.tempfile.gettempdir = lambda: {str(tmp_path)!r} + load_tmux._reap_stale_roots() + + print("REAPED" if not root.exists() else "KEPT") + """ + ) + + assert completed.returncode == 0, completed.stderr + assert "REAPED" in completed.stdout + + +def test_reaper_leaves_a_root_whose_owner_is_alive( + load_module_available: None, + tmp_path: pathlib.Path, +) -> None: + """A concurrent run's root is untouched. + + This is the property that makes the reaper safe to run unattended: stealing + another run's tmux server would be a worse failure than leaking this one. + """ + completed = _run_in_load_env( + f""" + root = pathlib.Path({str(tmp_path)!r}) / (load_tmux._ROOT_PREFIX + "live") + root.mkdir() + # This very process is the owner, so the identity check must match. + (root / "owner").write_text( + f"{{os.getpid()}} {{load_tmux._process_identity(os.getpid())}}" + ) + + load_tmux.tempfile.gettempdir = lambda: {str(tmp_path)!r} + load_tmux._reap_stale_roots() + + print("KEPT" if root.exists() else "REAPED") + """ + ) + + assert completed.returncode == 0, completed.stderr + assert "KEPT" in completed.stdout + + +def test_reaper_rejects_a_reused_pid( + load_module_available: None, + tmp_path: pathlib.Path, +) -> None: + """A live pid with a different start time is a reused number, not the owner. + + Without the start-time half of the identity, the reaper would skip a stale + root forever once the kernel handed its pid to something else -- or, worse, + a bare-pid *match* would let it reap a live run. + """ + completed = _run_in_load_env( + f""" + root = pathlib.Path({str(tmp_path)!r}) / (load_tmux._ROOT_PREFIX + "reused") + root.mkdir() + # A running pid, but stamped with a start time that is not its own. + (root / "owner").write_text(f"{{os.getpid()}} 1") + + load_tmux.tempfile.gettempdir = lambda: {str(tmp_path)!r} + load_tmux._reap_stale_roots() + + print("REAPED" if not root.exists() else "KEPT") + """ + ) + + assert completed.returncode == 0, completed.stderr + assert "REAPED" in completed.stdout + + +def test_process_identity_is_absent_for_a_dead_pid( + load_module_available: None, +) -> None: + """The identity probe reports absence rather than raising.""" + completed = _run_in_load_env( + """ + print("SELF", load_tmux._process_identity(os.getpid()) is not None) + print("DEAD", load_tmux._process_identity(4194304) is None) + """ + ) + + assert completed.returncode == 0, completed.stderr + assert "SELF True" in completed.stdout + assert "DEAD True" in completed.stdout + + +def test_hold_command_outlasts_any_plausible_run( + load_module_available: None, +) -> None: + """The pane's holding command bounds how long a run can last. + + When it exits the window closes, the last window closing ends the session, + and the server goes with it -- ``destroy-unattached off`` only survives + *detach*. So this value is a run-duration ceiling, and a short one would + kill a long ``--duration`` mid-flight. + """ + completed = _run_in_load_env( + """ + print("HOLD", load_tmux._HOLD_COMMAND) + """ + ) + + assert completed.returncode == 0, completed.stderr + seconds = int(completed.stdout.split("HOLD sleep ")[1].split()[0]) + assert seconds >= 3600, "a run longer than the hold command loses its server" From a7c95a490426a53fdef690ebe493e07b4b6a1355 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Fri, 21 Aug 2026 14:43:19 -0500 Subject: [PATCH 19/24] Bench(docs[lgtm]): State how long a load run can be why: The shipped scenarios run 10 and 16 seconds, so nothing in the recipes approaches a limit and a reader has no reason to suspect one. A long run used to climb in memory until it ended in swap rather than in a clean error, which presents as a CPU spike that never recovers. Bisecting it showed removing the OTel sink left the growth unchanged while removing rampa's per-iteration recording cut it by 70%: the cause was rampa's runner buffering every sample for outputs that were never configured, and it is fixed upstream. what: - State that duration is safe, and name the version the growth belongs to, so a climbing run has an obvious first thing to check - Explain why the pane's holding command outlasts any plausible duration --- scripts/lgtm/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md index ae1bb0ef0..f36827f55 100644 --- a/scripts/lgtm/README.md +++ b/scripts/lgtm/README.md @@ -249,6 +249,25 @@ Going through the sink instead means it lands under the same metric names and the same branch, worktree, and spike labels as every other run, and the two are directly comparable. +### How long a run can be + +The shipped scenarios run 10 and 16 seconds, but a long `--duration` is safe: +a steady run holds about 100 MB resident whatever its length, growing by +roughly 100 bytes per iteration rather than accumulating. + +That depends on the rampa version. Through 0.0.1a1, `rampa run` attached its +sample buffer unconditionally, so every metric sample was retained for the +whole run even when no configured output would read it. On this workload that +was about 1.75 KB per iteration -- 4 MB/s, climbing linearly with no plateau to +2.25 GB after nine minutes, which on a memory-capped host ends in swap rather +than a clean error. If a run's memory climbs steadily, check whether rampa +predates that fix before looking anywhere else. + +The tmux server's pane runs a holding command that outlasts any plausible +`--duration` by design. If it exited first the window would close, the last +window closing would end the session, and the server would go with it +mid-run -- `destroy-unattached off` only survives *detach*. + ## Why the acceptance check exists A dashboard that renders is not a dashboard that works. A panel whose query From 0d515a98ba41ec392f4b4e3e63c480646f3510bf Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sat, 22 Aug 2026 20:34:07 -0500 Subject: [PATCH 20/24] Bench(fix[lgtm]): Type-check the scripts the way CI does why: CI runs `mypy .`, which covers `scripts/`; the configured file list that plain `mypy` uses does not, so these went unchecked on developer machines and failed the gate the first time this branch had a pull request. Two of them were real: telemetry and the load scenario still reached command_count through control_mode, which re-exports it rather than defining it, so the import resolved at runtime but not under strict checking. The rest were missing type arguments and a literal inferred as list[object]. what: - Take command_count from libtmux.engines.base, its definition site - Give the acceptance script's dict annotations their arguments, and name the decoded JSON body rather than returning Any - Annotate the error thresholds so the generated panel keeps its shape - Narrow the git-dir probe explicitly, since bool() does not narrow Optional - Pass rampa a timedelta rather than a duration string: its field coerces the string through a validator, so the value is identical and the annotation stops lying - Name the async lane factory separately from the sync one, and mark the optional profiler import untyped --- scripts/lgtm/generate_dashboards.py | 2 +- scripts/lgtm/identity.py | 6 ++++-- scripts/lgtm/load_tmux.py | 10 ++++++---- scripts/lgtm/telemetry.py | 2 +- scripts/otel_acceptance.py | 13 +++++++++---- scripts/otel_smoke.py | 6 +++--- 6 files changed, 24 insertions(+), 15 deletions(-) diff --git a/scripts/lgtm/generate_dashboards.py b/scripts/lgtm/generate_dashboards.py index dc43d4ea4..65a097da4 100644 --- a/scripts/lgtm/generate_dashboards.py +++ b/scripts/lgtm/generate_dashboards.py @@ -73,7 +73,7 @@ def quantile_by(quantile: float, label: str) -> str: ) -ERR_THRESHOLDS = [ +ERR_THRESHOLDS: list[dict[str, t.Any]] = [ {"color": "green", "value": None}, {"color": "orange", "value": 1}, {"color": "red", "value": 5}, diff --git a/scripts/lgtm/identity.py b/scripts/lgtm/identity.py index 7bfd82e78..57dbf67d2 100644 --- a/scripts/lgtm/identity.py +++ b/scripts/lgtm/identity.py @@ -134,8 +134,10 @@ def vcs_attributes(root: pathlib.Path | None = None) -> dict[str, str]: # the label too, since that is how sibling checkouts are told apart. git_dir = _git(root, "rev-parse", "--git-dir") checkout = pathlib.Path(toplevel).name - linked = bool(git_dir) and ( - pathlib.Path(git_dir[0]).resolve() != pathlib.Path(common_dir).resolve() + linked = ( + git_dir is not None + and bool(git_dir) + and (pathlib.Path(git_dir[0]).resolve() != pathlib.Path(common_dir).resolve()) ) if linked or checkout != attributes["vcs.repository.name"]: attributes["libtmux.worktree"] = checkout diff --git a/scripts/lgtm/load_tmux.py b/scripts/lgtm/load_tmux.py index 5e9a06986..6a475c16f 100644 --- a/scripts/lgtm/load_tmux.py +++ b/scripts/lgtm/load_tmux.py @@ -17,6 +17,7 @@ import asyncio import contextlib +import datetime import os import pathlib import shutil @@ -33,13 +34,13 @@ import rampa import telemetry +from libtmux.engines.base import command_count from libtmux.experimental.engines import ( AsyncControlModeEngine, AsyncSubprocessEngine, instrument, ) from libtmux.experimental.engines.base import CommandRequest, CommandSeparator -from libtmux.experimental.engines.control_mode import command_count from libtmux.server import Server # Transports this scenario can drive, and how to build each. @@ -248,8 +249,9 @@ def _engine() -> t.Any: spike=os.environ.get("LIBTMUX_SPIKE"), ) _STATE["signals"] = signals + engine_factory = t.cast("t.Callable[[t.Any], t.Any]", factory) _STATE["engine"] = instrument( - factory(server), + engine_factory(server), telemetry.OTelSink( signals.tracer, signals.meter, LANE, signals.metric_labels ), @@ -284,8 +286,8 @@ async def steady(worker: rampa.Worker) -> None: @rampa.scenario( executor="ramping-arrival-rate", stages=[ - rampa.Stage(duration="8s", target=200), - rampa.Stage(duration="8s", target=1200), + rampa.Stage(duration=datetime.timedelta(seconds=8), target=200), + rampa.Stage(duration=datetime.timedelta(seconds=8), target=1200), ], pre_allocated_vus=32, max_vus=256, diff --git a/scripts/lgtm/telemetry.py b/scripts/lgtm/telemetry.py index 0cce5da5a..1e7e6f4c5 100644 --- a/scripts/lgtm/telemetry.py +++ b/scripts/lgtm/telemetry.py @@ -38,7 +38,7 @@ from opentelemetry.sdk.trace import SpanProcessor, TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor -from libtmux.experimental.engines.control_mode import command_count +from libtmux.engines.base import command_count if t.TYPE_CHECKING: from libtmux.experimental.engines.base import CommandRequest, CommandResult diff --git a/scripts/otel_acceptance.py b/scripts/otel_acceptance.py index cb3c7a031..591058d79 100644 --- a/scripts/otel_acceptance.py +++ b/scripts/otel_acceptance.py @@ -99,14 +99,17 @@ def expand(expr: str) -> str: ) -def fetch(url: str, params: dict[str, str], *, timeout: float = 30.0) -> dict: +def fetch( + url: str, params: dict[str, str], *, timeout: float = 30.0 +) -> dict[str, t.Any]: """GET *url* with *params* and decode the JSON body.""" query = urllib.parse.urlencode(params) with urllib.request.urlopen(f"{url}?{query}", timeout=timeout) as response: - return json.loads(response.read().decode()) + decoded: dict[str, t.Any] = json.loads(response.read().decode()) + return decoded -def _finite_series(rows: list[dict], *, ranged: bool) -> int: +def _finite_series(rows: list[dict[str, t.Any]], *, ranged: bool) -> int: """Count series carrying at least one real number. ``histogram_quantile`` over empty buckets yields NaN rather than an empty @@ -259,7 +262,9 @@ def answers(url: str) -> bool: return [name for name, url in probes if not answers(url)] -def check_panel(panel: dict, board: str, endpoints: Endpoints) -> list[Result]: +def check_panel( + panel: dict[str, t.Any], board: str, endpoints: Endpoints +) -> list[Result]: """Verify every target on one panel.""" results: list[Result] = [] title = panel.get("title", "") diff --git a/scripts/otel_smoke.py b/scripts/otel_smoke.py index 2fce76ce4..5fc39c1ef 100644 --- a/scripts/otel_smoke.py +++ b/scripts/otel_smoke.py @@ -154,7 +154,7 @@ def _profile_lane(lane: str, *, enabled: bool) -> t.Iterator[None]: if not enabled: yield return - import pyroscope + import pyroscope # type: ignore[import-untyped] with pyroscope.tag_wrapper({"tmux_lane": lane}): yield @@ -345,10 +345,10 @@ def main(argv: list[str] | None = None) -> int: ("subprocess-async", lambda: AsyncSubprocessEngine.for_server(server)), ("control-async", lambda: AsyncControlModeEngine.for_server(server)), ) - for lane, factory in async_lanes: + for lane, async_factory in async_lanes: counts = CountingSink() engine = instrument( - factory(), + async_factory(), counts, telemetry.OTelSink( signals.tracer, signals.meter, lane, signals.metric_labels From fd5babc63e97ae7b77585d9ce5eec7222a02a3a4 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sat, 22 Aug 2026 20:39:35 -0500 Subject: [PATCH 21/24] Bench(fix[lgtm]): Let mypy type-check the scripts without the otel group why: CI runs `mypy .`, which reaches `scripts/`, but installs only the default and dev groups -- the `otel` group is deliberately excluded so the ordinary gates stay lean. Every OpenTelemetry, pyroscope, and rampa import was therefore unresolvable in CI while resolving fine on a machine that had the group installed. what: - Allow those three imports to be missing, so the scripts type-check in both environments rather than only the one the developer happens to have - Relax subclassing-Any and untyped-decorator for the two modules that build on them, scoped per module: an inline ignore would itself be unused wherever the group is installed --- pyproject.toml | 24 ++++++++++++++++++++++++ scripts/otel_smoke.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 16b4ee6b3..0920799b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -186,6 +186,30 @@ files = [ "tests", ] +# The observability scripts import third-party packages from the optional +# `otel` group, which CI does not install so the ordinary gates stay lean. +# `mypy .` still reaches those scripts, so their imports must be allowed to +# be absent. `rampa` ships no stubs either way. +[[tool.mypy.overrides]] +module = [ + "opentelemetry.*", + "pyroscope", + "rampa", +] +ignore_missing_imports = true + +# Those same packages ship no stubs, so anything derived from them is `Any`: +# a SpanProcessor subclass and rampa's scenario decorators. Relaxing the two +# strictness flags here is scoped to these scripts and, unlike an inline +# ignore, stays correct whether or not the optional group is installed. +[[tool.mypy.overrides]] +module = [ + "telemetry", + "load_tmux", +] +disallow_subclassing_any = false +disallow_untyped_decorators = false + [tool.ty.environment] python-version = "3.10" diff --git a/scripts/otel_smoke.py b/scripts/otel_smoke.py index 5fc39c1ef..6fabe5b5a 100644 --- a/scripts/otel_smoke.py +++ b/scripts/otel_smoke.py @@ -154,7 +154,7 @@ def _profile_lane(lane: str, *, enabled: bool) -> t.Iterator[None]: if not enabled: yield return - import pyroscope # type: ignore[import-untyped] + import pyroscope with pyroscope.tag_wrapper({"tmux_lane": lane}): yield From 42c45646fd28a5b622115730d1f2d8d5da8345d8 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sat, 22 Aug 2026 21:02:05 -0500 Subject: [PATCH 22/24] Bench(fix[lgtm]): Skip the optional-dependency tests on what they import why: Both tests guard on `opentelemetry`, but that namespace package arrives transitively, so the guard passes wherever libtmux is installed. What the modules under test actually import is `opentelemetry.sdk`, the OTLP exporters, and rampa, all of which ship only in the optional `otel` group -- which CI does not install. The guard therefore never fired and the tests failed on import instead of skipping, which was the stated intent. what: - Guard on `opentelemetry.sdk` rather than the namespace package - Guard the load scenario on rampa as well, since it imports both Verified in both directions: without the group the two skip, and with it installed they still run and pass rather than being silently disabled. --- tests/test_lgtm_dashboards.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_lgtm_dashboards.py b/tests/test_lgtm_dashboards.py index b6718d1e2..be7e240d1 100644 --- a/tests/test_lgtm_dashboards.py +++ b/tests/test_lgtm_dashboards.py @@ -277,7 +277,11 @@ def test_telemetry_doctests_execute() -> None: """ import doctest - pytest.importorskip("opentelemetry", reason="otel dependency group not installed") + # The namespace package arrives transitively; the SDK is what this module + # actually imports, and it only ships in the optional group. + pytest.importorskip( + "opentelemetry.sdk", reason="otel dependency group not installed" + ) spec = importlib.util.spec_from_file_location("telemetry", _LGTM / "telemetry.py") assert spec is not None @@ -463,7 +467,10 @@ def test_a_failing_load_setup_is_attempted_once( input into thousands of servers and enough threads to saturate the machine. It did exactly that once, which is why this is pinned. """ - pytest.importorskip("opentelemetry", reason="otel dependency group not installed") + pytest.importorskip( + "opentelemetry.sdk", reason="otel dependency group not installed" + ) + pytest.importorskip("rampa", reason="otel dependency group not installed") monkeypatch.syspath_prepend(str(_LGTM)) spec = importlib.util.spec_from_file_location("load_tmux", _LGTM / "load_tmux.py") From b265f31a4300fc97de56b886c8e09f4032298a3d Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 23 Aug 2026 10:28:32 -0500 Subject: [PATCH 23/24] Bench(refactor[lgtm]): Move the otel scripts into the stack they belong to `scripts/lgtm/` holds the observability stack -- its compose files, dashboard generator, telemetry wiring, and load driver. The smoke run and the acceptance check sat outside it under an `otel_` prefix, though they are the two entry points that stack exists to serve: `acceptance.py` already documented itself in terms of `scripts.lgtm.generate_dashboards`, and `generate_dashboards` returns the reference. scripts/otel_smoke.py -> scripts/lgtm/smoke.py scripts/otel_acceptance.py -> scripts/lgtm/acceptance.py Inside the directory the prefix was saying what the directory already says, so it goes, matching `identity.py`, `telemetry.py`, and `load_tmux.py` beside them. Moving one level deeper broke two paths that had been counting directories, both silently: - `acceptance.py` derived the repository root with `parent.parent`, which now lands on `scripts/`, pointing every dashboard and helper lookup one level wrong. - `smoke.py` put `__file__.parent / "lgtm"` on `sys.path` to import `telemetry`; from inside `lgtm/` that names a directory that does not exist. It now adds its own directory, the idiom `load_tmux.py` already uses. `--help` used to say `otel_smoke.py`, which told a reader where to look. Bare `smoke.py` does not, so both parsers name their path, as the orchestration scripts do. Verified: both run under the invocation their `just` recipes use, every derived path resolves, and the lgtm tests pass. --- CHANGES | 2 +- justfile | 6 +++--- scripts/lgtm/README.md | 4 ++-- scripts/{otel_acceptance.py => lgtm/acceptance.py} | 10 ++++++---- scripts/lgtm/generate_dashboards.py | 4 ++-- scripts/lgtm/load_tmux.py | 2 +- scripts/{otel_smoke.py => lgtm/smoke.py} | 6 +++--- tests/test_lgtm_dashboards.py | 4 ++-- 8 files changed, 20 insertions(+), 18 deletions(-) rename scripts/{otel_acceptance.py => lgtm/acceptance.py} (97%) rename scripts/{otel_smoke.py => lgtm/smoke.py} (98%) diff --git a/CHANGES b/CHANGES index adfcfb55c..eeb5a2983 100644 --- a/CHANGES +++ b/CHANGES @@ -286,7 +286,7 @@ engine instrumentation seam, so the exporters are ordinary sinks and libtmux itself gains no OpenTelemetry dependency. The Grafana dashboards under `scripts/lgtm/dashboards/` are generated rather -than hand-edited, and `scripts/otel_acceptance.py` runs each panel's own query +than hand-edited, and `scripts/lgtm/acceptance.py` runs each panel's own query and fails naming any panel that came back empty. See `scripts/lgtm/README.md`. Every run is stamped with its branch, revision, repository, worktree, and an diff --git a/justfile b/justfile index be0abd78d..241b4aef9 100644 --- a/justfile +++ b/justfile @@ -173,17 +173,17 @@ otel-dashboards: # Drive a real tmux workload through the engine seam into LGTM [group: 'otel'] otel-smoke *args: - uv run --group otel python scripts/otel_smoke.py {{ args }} + uv run --group otel python scripts/lgtm/smoke.py {{ args }} # Verify every dashboard panel's own queries return data [group: 'otel'] otel-acceptance *args: - uv run --group otel python scripts/otel_acceptance.py {{ args }} + uv run --group otel python scripts/lgtm/acceptance.py {{ args }} # Start the stack, run the workload, then verify every panel end to end [group: 'otel'] otel-verify: - uv run --group otel python scripts/otel_acceptance.py --start-stack --smoke + uv run --group otel python scripts/lgtm/acceptance.py --start-stack --smoke # Drive the engines under a load shape (ramping arrival rate) via rampa [group: 'otel'] diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md index f36827f55..bb085ca5d 100644 --- a/scripts/lgtm/README.md +++ b/scripts/lgtm/README.md @@ -82,7 +82,7 @@ $ just otel-ports ## What the workload emits -`scripts/otel_smoke.py` runs every transport — subprocess and control mode, +`scripts/lgtm/smoke.py` runs every transport — subprocess and control mode, sync and async — against a throwaway tmux server, and emits four signals: Metrics are `tmux_requests_total`, `tmux_commands_total`, `tmux_inlined_total`, @@ -273,7 +273,7 @@ mid-run -- `destroy-unattached off` only survives *detach*. A dashboard that renders is not a dashboard that works. A panel whose query returns nothing looks exactly like a panel reporting a healthy zero. -`scripts/otel_acceptance.py` reads the generated JSON, expands the template +`scripts/lgtm/acceptance.py` reads the generated JSON, expands the template variables the way Grafana would, runs every panel's own query against Prometheus, Loki, Tempo, or Pyroscope, and fails naming any panel that returned nothing. Because it reads the dashboards themselves, a panel added to the diff --git a/scripts/otel_acceptance.py b/scripts/lgtm/acceptance.py similarity index 97% rename from scripts/otel_acceptance.py rename to scripts/lgtm/acceptance.py index 591058d79..9d03b7fda 100644 --- a/scripts/otel_acceptance.py +++ b/scripts/lgtm/acceptance.py @@ -28,7 +28,7 @@ import urllib.parse import urllib.request -ROOT = pathlib.Path(__file__).resolve().parent.parent +ROOT = pathlib.Path(__file__).resolve().parents[2] DASHBOARDS = ROOT / "scripts" / "lgtm" / "dashboards" SERVICE = "libtmux-engines" @@ -327,7 +327,9 @@ def check_until(endpoints: Endpoints, timeout: float, poll: float) -> list[Resul def main(argv: list[str] | None = None) -> int: """Run the acceptance sweep and print a per-panel report.""" - parser = argparse.ArgumentParser(description=__doc__) + parser = argparse.ArgumentParser( + prog="scripts/lgtm/acceptance.py", description=__doc__ + ) parser.add_argument("--prometheus", default="http://127.0.0.1:9099") parser.add_argument("--loki", default="http://127.0.0.1:3100") parser.add_argument("--tempo", default="http://127.0.0.1:3200") @@ -336,7 +338,7 @@ def main(argv: list[str] | None = None) -> int: "--start-stack", action="store_true", help="run scripts/lgtm/up.sh first" ) parser.add_argument( - "--smoke", action="store_true", help="run scripts/otel_smoke.py first" + "--smoke", action="store_true", help="run scripts/lgtm/smoke.py first" ) parser.add_argument( "--settle", @@ -362,7 +364,7 @@ def main(argv: list[str] | None = None) -> int: subprocess.run([str(ROOT / "scripts" / "lgtm" / "up.sh")], check=True) if args.smoke: subprocess.run( - [sys.executable, str(ROOT / "scripts" / "otel_smoke.py")], check=True + [sys.executable, str(ROOT / "scripts" / "lgtm" / "smoke.py")], check=True ) if args.start_stack or args.smoke: time.sleep(args.settle) diff --git a/scripts/lgtm/generate_dashboards.py b/scripts/lgtm/generate_dashboards.py index 65a097da4..e38d59dc7 100644 --- a/scripts/lgtm/generate_dashboards.py +++ b/scripts/lgtm/generate_dashboards.py @@ -9,8 +9,8 @@ start and ``tests/test_lgtm_dashboards.py`` fails if the committed copy differs, so the two cannot silently diverge. -Every panel must be backed by telemetry ``scripts/otel_smoke.py`` actually -emits. ``scripts/otel_acceptance.py`` executes each panel's own queries and +Every panel must be backed by telemetry ``scripts/lgtm/smoke.py`` actually +emits. ``scripts/lgtm/acceptance.py`` executes each panel's own queries and fails on any that returns nothing, which is what keeps a board honest. """ diff --git a/scripts/lgtm/load_tmux.py b/scripts/lgtm/load_tmux.py index 6a475c16f..1fda2b277 100644 --- a/scripts/lgtm/load_tmux.py +++ b/scripts/lgtm/load_tmux.py @@ -1,6 +1,6 @@ """Load-shape the tmux engines with rampa, and export the result to LGTM. -``otel_smoke.py`` answers "does telemetry flow" by running flat out for a fixed +``smoke.py`` answers "does telemetry flow" by running flat out for a fixed duration. That is the wrong shape for asking where a transport stops keeping up, because a closed loop of N workers slows down with the system: offered load falls as latency rises, and the graph bends politely instead of breaking. diff --git a/scripts/otel_smoke.py b/scripts/lgtm/smoke.py similarity index 98% rename from scripts/otel_smoke.py rename to scripts/lgtm/smoke.py index 6fabe5b5a..a9b165aec 100644 --- a/scripts/otel_smoke.py +++ b/scripts/lgtm/smoke.py @@ -28,7 +28,7 @@ import typing as t import uuid -sys.path.insert(0, str(pathlib.Path(__file__).parent / "lgtm")) +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) import telemetry @@ -46,7 +46,7 @@ from libtmux.experimental.engines.instrumentation import CountingSink from libtmux.server import Server -logger = logging.getLogger("libtmux.otel_smoke") +logger = logging.getLogger("libtmux.lgtm.smoke") PLAIN = CommandRequest.from_args("list-panes", "-a", "-F", "#{pane_id}") LISTING = CommandRequest.from_args("list-windows", "-a", "-F", "#{window_id}") @@ -253,7 +253,7 @@ def lane_totals(counts: CountingSink) -> dict[str, int]: def main(argv: list[str] | None = None) -> int: """Run every lane under full telemetry and print the local counts.""" - parser = argparse.ArgumentParser(description=__doc__) + parser = argparse.ArgumentParser(prog="scripts/lgtm/smoke.py", description=__doc__) parser.add_argument("--run-id", default=f"smoke-{uuid.uuid4().hex[:8]}") parser.add_argument( "--spike", diff --git a/tests/test_lgtm_dashboards.py b/tests/test_lgtm_dashboards.py index be7e240d1..a28e1f5b8 100644 --- a/tests/test_lgtm_dashboards.py +++ b/tests/test_lgtm_dashboards.py @@ -1,7 +1,7 @@ """Structural contracts for the generated Grafana dashboards. These run offline. They cannot tell whether a panel has data -- that is -``scripts/otel_acceptance.py`` against a live stack -- but they do keep the +``scripts/lgtm/acceptance.py`` against a live stack -- but they do keep the committed JSON honest about its generator and about the datasources it binds to, which is where a board rots silently. """ @@ -135,7 +135,7 @@ def test_acceptance_expands_every_template_variable() -> None: through as text and match nothing. """ spec = importlib.util.spec_from_file_location( - "otel_acceptance", _ROOT / "scripts" / "otel_acceptance.py" + "lgtm_acceptance", _ROOT / "scripts" / "lgtm" / "acceptance.py" ) assert spec is not None assert spec.loader is not None From 3faee623c1844a7c6c6b8f014fbbaffabb66a108 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 23 Aug 2026 12:55:54 -0500 Subject: [PATCH 24/24] Tests(refactor): Mirror the lgtm tests under tests/scripts Both cover `scripts/lgtm/`, so they sit beside it. Their repository-root lookups count directories, and the counts move with them. The dashboard generator and the stack's README each pointed at the dashboard test by its old name, which is the pairing this layout now makes checkable from the path. With these, no branch in the stack carries a `scripts/bench_*` or a `tests/test_bench_*`: one convention, everywhere, rather than one that arrives partway up. --- scripts/lgtm/README.md | 2 +- scripts/lgtm/generate_dashboards.py | 2 +- tests/scripts/lgtm/__init__.py | 1 + .../lgtm/test_dashboards.py} | 2 +- tests/{test_lgtm_load.py => scripts/lgtm/test_load.py} | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 tests/scripts/lgtm/__init__.py rename tests/{test_lgtm_dashboards.py => scripts/lgtm/test_dashboards.py} (99%) rename tests/{test_lgtm_load.py => scripts/lgtm/test_load.py} (99%) diff --git a/scripts/lgtm/README.md b/scripts/lgtm/README.md index bb085ca5d..fb33a910d 100644 --- a/scripts/lgtm/README.md +++ b/scripts/lgtm/README.md @@ -210,7 +210,7 @@ changing the generator: $ just otel-dashboards ``` -`up.sh` regenerates on every start, and `tests/test_lgtm_dashboards.py` fails if +`up.sh` regenerates on every start, and `tests/scripts/lgtm/test_dashboards.py` fails if the committed JSON differs from what the generator produces, so the two cannot diverge quietly. Editing a board in the Grafana UI is fine for exploring; move the change into the generator to keep it. diff --git a/scripts/lgtm/generate_dashboards.py b/scripts/lgtm/generate_dashboards.py index e38d59dc7..d19b0e69d 100644 --- a/scripts/lgtm/generate_dashboards.py +++ b/scripts/lgtm/generate_dashboards.py @@ -6,7 +6,7 @@ list of panel calls and :class:`Board` does the grid math. The generated JSON is committed. ``scripts/lgtm/up.sh`` regenerates it on every -start and ``tests/test_lgtm_dashboards.py`` fails if the committed copy differs, +start and ``tests/scripts/lgtm/test_dashboards.py`` fails if the committed copy differs, so the two cannot silently diverge. Every panel must be backed by telemetry ``scripts/lgtm/smoke.py`` actually diff --git a/tests/scripts/lgtm/__init__.py b/tests/scripts/lgtm/__init__.py new file mode 100644 index 000000000..4b0d4255c --- /dev/null +++ b/tests/scripts/lgtm/__init__.py @@ -0,0 +1 @@ +"""Tests for scripts/lgtm/.""" diff --git a/tests/test_lgtm_dashboards.py b/tests/scripts/lgtm/test_dashboards.py similarity index 99% rename from tests/test_lgtm_dashboards.py rename to tests/scripts/lgtm/test_dashboards.py index a28e1f5b8..f20176f51 100644 --- a/tests/test_lgtm_dashboards.py +++ b/tests/scripts/lgtm/test_dashboards.py @@ -18,7 +18,7 @@ import pytest -_ROOT = pathlib.Path(__file__).parents[1] +_ROOT = pathlib.Path(__file__).parents[3] _LGTM = _ROOT / "scripts" / "lgtm" _DASHBOARDS = _LGTM / "dashboards" diff --git a/tests/test_lgtm_load.py b/tests/scripts/lgtm/test_load.py similarity index 99% rename from tests/test_lgtm_load.py rename to tests/scripts/lgtm/test_load.py index 5e95dbb02..6084d5c78 100644 --- a/tests/test_lgtm_load.py +++ b/tests/scripts/lgtm/test_load.py @@ -20,7 +20,7 @@ import pytest -_ROOT = pathlib.Path(__file__).parents[1] +_ROOT = pathlib.Path(__file__).parents[3] _PREAMBLE = (