From 84762dab1af255bb512c482507ec6feb1e0f45eb Mon Sep 17 00:00:00 2001 From: venkateshsakamuri-lab Date: Fri, 7 Aug 2026 16:21:49 +0530 Subject: [PATCH] fix: make self-host Agent tab and dashboards work end-to-end Wire Hermes via host.docker.internal, profile/switch + cookies, and setup-agent.sh so Agent chat and AI dashboards work on a fresh self-host install. Co-authored-by: Cursor --- .env.example | 8 + .gitignore | 2 + README.md | 5 +- agent/install.sh | 101 +++++-- .../com/dbaagent/service/AgentChatClient.java | 32 +- .../service/scheduler/BrainJobsService.java | 8 +- .../scheduler/BrainJobsServiceTest.java | 32 ++ docker-compose.yml | 13 + docker/nginx/default.conf | 24 +- docs/root/SELF_HOST_GUIDE.md | 129 ++++++-- scripts/self-host/e2e-agent-check.py | 219 ++++++++++++++ scripts/self-host/install.sh | 21 ++ scripts/self-host/setup-agent.sh | 284 ++++++++++++++++++ scripts/self-host/smoke-test.sh | 92 +++++- scripts/self-host/status.sh | 15 + src/components/AgentChat/AgentChatPanel.jsx | 3 + src/lib/api/agentClient.js | 7 + vite.config.js | 4 +- 18 files changed, 944 insertions(+), 55 deletions(-) create mode 100755 scripts/self-host/e2e-agent-check.py create mode 100755 scripts/self-host/setup-agent.sh diff --git a/.env.example b/.env.example index d8fee45..62d3cf3 100644 --- a/.env.example +++ b/.env.example @@ -128,6 +128,14 @@ DEEPSQL_EMBEDDING_MODEL=text-embedding-3-large # profile which defaults to false. EMBEDDING_FAIL_OPEN=false +# ── Hermes agent (Agent tab + AI dashboards) ──────────────────────────────── +# Backend → Hermes for dashboards / Slack / CLI. Compose defaults this; only +# override if Hermes listens elsewhere. Requires scripts/self-host/setup-agent.sh +# (also run by install.sh unless DEEPSQL_SKIP_AGENT_SETUP=1). +#AGENT_WEBUI_URL=http://host.docker.internal:8787 +#DEEPSQL_SKIP_AGENT_SETUP=0 +#DEEPSQL_SMOKE_AGENT=1 + # ── The /api/llm/v1 agent gateway ─────────────────────────────────────────── # The DeepSQL CLI agent (`@deepsql/mcp`) points its model at /api/llm/v1 # and authenticates with a DeepSQL token; the backend forwards those calls diff --git a/.gitignore b/.gitignore index 01a55db..46a2080 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,5 @@ optd-sidecar/target/ .idea/ .vscode/ *.iml +.local-admin-credentials +.local-mcp-token diff --git a/README.md b/README.md index dcec6c4..374c642 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,10 @@ is rejected. Changing width means migrating the column. The installer generates your JWT secret, the credential-vault encryption key and the vault DB password, prompts for the first admin account, builds both images, starts the stack, and -verifies pgvector is live. +verifies pgvector is live. Unless you set `DEEPSQL_SKIP_AGENT_SETUP=1`, it also runs +[`scripts/self-host/setup-agent.sh`](scripts/self-host/setup-agent.sh) to install Hermes under +`~/.hermes/`, wire DeepSQL MCP, and start the webui on `0.0.0.0:8787` (required for the +**Agent** tab and AI dashboard generation). **The first build takes several minutes** — it compiles the Spring Boot backend with Maven inside the container and bundles the frontend with Vite. It has not hung. Later builds reuse diff --git a/agent/install.sh b/agent/install.sh index dd67978..6943293 100755 --- a/agent/install.sh +++ b/agent/install.sh @@ -3,7 +3,7 @@ # Idempotent: safe to re-run. Source of truth is this repo's agent/ dir. # # Configures: -# - model: existing Azure OpenAI gpt-5.4 via its OpenAI-compatible v1 endpoint +# - model: from DEEPSQL_CHAT_* (or legacy AZURE_OPENAI_*) via OpenAI-compatible endpoint # - mcp_servers.deepsql: the repo's DeepSQL MCP server (read-only DBA tools) # - skills.external_dirs: this repo's agent/skills (source of truth) # - approvals.mode: smart @@ -11,47 +11,107 @@ # - disables host-affecting toolsets (terminal/file/code/browser/computer_use) # # Secrets are read from the environment (or the repo .env), never committed: -# AZURE_OPENAI_KEY, AZURE_OPENAI_ENDPOINT (endpoint defaults to the repo value) +# DEEPSQL_CHAT_API_KEY, DEEPSQL_CHAT_ENDPOINT, DEEPSQL_CHAT_MODEL +# (legacy fallback: AZURE_OPENAI_KEY, AZURE_OPENAI_ENDPOINT) # # Upstream note: HERMES_HOME / hermes-agent / hermes CLI are contracts of the # Nous Hermes Agent runtime this customization runs on — do not rename those. set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}" +# Reject a nested profile home inherited from a live Hermes process. +if [[ "${HERMES_HOME:-}" == */profiles/* ]]; then + unset HERMES_HOME +fi +HERMES_HOME="${DEEPSQL_HERMES_HOME:-${HERMES_HOME:-$HOME/.hermes}}" AGENT_DIR="${HERMES_AGENT_DIR:-$HERMES_HOME/hermes-agent}" -VENV_PY="$AGENT_DIR/.venv/bin/python" -# Load repo .env for Azure creds if not already in the environment. +# Prefer the same interpreter order as scripts/self-host/setup-agent.sh +if [[ -x "$AGENT_DIR/venv/bin/python" ]]; then + VENV_PY="$AGENT_DIR/venv/bin/python" +elif [[ -x "$AGENT_DIR/.venv/bin/python" ]]; then + VENV_PY="$AGENT_DIR/.venv/bin/python" +else + echo "Agent venv not found under $AGENT_DIR — run scripts/self-host/setup-agent.sh first." >&2 + exit 1 +fi + +# Load repo .env for LLM creds if not already in the environment. if [[ -f "$REPO_ROOT/.env" ]]; then set -a; . "$REPO_ROOT/.env"; set +a; fi -: "${AZURE_OPENAI_KEY:?Set AZURE_OPENAI_KEY (or add it to repo .env)}" -AZURE_ENDPOINT_HOST="$(printf '%s' "${AZURE_OPENAI_ENDPOINT:-https://your-resource.cognitiveservices.azure.com/}" | sed -E 's#https?://##; s#/.*##; s#\.cognitiveservices\.azure\.com#.openai.azure.com#')" -BASE_URL="https://${AZURE_ENDPOINT_HOST}/openai/v1" -[[ -x "$VENV_PY" ]] || { echo "Agent venv not found at $VENV_PY — install the agent runtime first."; exit 1; } +# Prefer the same BYO-LLM vars the Spring backend uses. Fall back to legacy +# AZURE_OPENAI_* for older checkouts. +API_KEY="${DEEPSQL_CHAT_API_KEY:-${AZURE_OPENAI_KEY:-}}" +ENDPOINT="${DEEPSQL_CHAT_ENDPOINT:-${AZURE_OPENAI_ENDPOINT:-}}" +MODEL="${DEEPSQL_CHAT_MODEL:-gpt-5.4}" + +if [[ -z "$API_KEY" ]]; then + echo "Error: set DEEPSQL_CHAT_API_KEY (or AZURE_OPENAI_KEY) in the environment or $REPO_ROOT/.env" >&2 + exit 1 +fi +if [[ -z "$ENDPOINT" ]]; then + echo "Error: set DEEPSQL_CHAT_ENDPOINT (or AZURE_OPENAI_ENDPOINT)." >&2 + exit 1 +fi + +# Normalize to an OpenAI-compatible …/openai/v1 or …/v1 base URL. +# Azure Cognitive Services / Azure OpenAI hosts need /openai/v1; plain OpenAI +# and OpenAI-compatible servers already expose /v1. +normalize_base_url() { + local ep="$1" + ep="${ep%/}" + if [[ "$ep" == *"/openai/v1" || "$ep" == *"/v1" ]]; then + printf '%s' "$ep" + return + fi + if [[ "$ep" == *".cognitiveservices.azure.com"* || "$ep" == *".openai.azure.com"* || "$ep" == *".azure-api.net"* ]]; then + printf '%s/openai/v1' "$ep" + return + fi + printf '%s/v1' "$ep" +} +BASE_URL="$(normalize_base_url "$ENDPOINT")" + +BACKEND_PORT="${DEEPSQL_BACKEND_PORT:-8080}" echo "→ Repo: $REPO_ROOT" echo "→ Agent home: $HERMES_HOME" -echo "→ Model base: $BASE_URL" +echo "→ Model: $MODEL @ $BASE_URL" -# Deep-merge the DBA config blocks into ~/.hermes/config.yaml (PyYAML ships with the agent). -REPO_ROOT="$REPO_ROOT" BASE_URL="$BASE_URL" AZURE_OPENAI_KEY="$AZURE_OPENAI_KEY" \ -HERMES_HOME="$HERMES_HOME" "$VENV_PY" - <<'PY' +REPO_ROOT="$REPO_ROOT" BASE_URL="$BASE_URL" API_KEY="$API_KEY" MODEL="$MODEL" \ +BACKEND_PORT="$BACKEND_PORT" HERMES_HOME="$HERMES_HOME" "$VENV_PY" - <<'PY' import os, yaml, pathlib home = pathlib.Path(os.environ["HERMES_HOME"]); repo = os.environ["REPO_ROOT"] cfg_path = home / "config.yaml" cfg = yaml.safe_load(cfg_path.read_text()) if cfg_path.exists() else {} cfg = cfg or {} cfg["model"] = { - "default": "gpt-5.4", "provider": "custom", - "base_url": os.environ["BASE_URL"], "api_key": os.environ["AZURE_OPENAI_KEY"], - "api_mode": "chat_completions", "context_length": 272000, + "default": os.environ["MODEL"], + "provider": "custom", + "base_url": os.environ["BASE_URL"], + "api_key": os.environ["API_KEY"], + "api_mode": "chat_completions", + "context_length": 272000, +} +cfg.setdefault("providers", {})["custom"] = { + "base_url": os.environ["BASE_URL"], + "api_key": os.environ["API_KEY"], +} +# Keep an existing DEEPSQL_AUTH_TOKEN if a prior setup-agent run wrote one into +# the root config; otherwise leave token unset — setup-agent.sh provisions the +# per-user profile with a minted token. +existing_env = ((cfg.get("mcp_servers") or {}).get("deepsql") or {}).get("env") or {} +mcp_env = { + "DEEPSQL_API_BASE_URL": f"http://localhost:{os.environ['BACKEND_PORT']}/api/", + "DEEPSQL_MCP_USER_ID": existing_env.get("DEEPSQL_MCP_USER_ID", "deepsql-agent"), + "DEEPSQL_MCP_PROJECT_ID": existing_env.get("DEEPSQL_MCP_PROJECT_ID", "deepsql-agent"), } +if existing_env.get("DEEPSQL_AUTH_TOKEN"): + mcp_env["DEEPSQL_AUTH_TOKEN"] = existing_env["DEEPSQL_AUTH_TOKEN"] cfg.setdefault("mcp_servers", {})["deepsql"] = { "command": "node", "args": [f"{repo}/mcp/deepsql-phase1-server.js"], - "env": {"DEEPSQL_API_BASE_URL": "http://localhost:8080/api/", - "DEEPSQL_MCP_USER_ID": "deepsql-agent", "DEEPSQL_MCP_PROJECT_ID": "deepsql-agent"}, + "env": mcp_env, } cfg.setdefault("skills", {})["external_dirs"] = [f"{repo}/agent/skills"] cfg.setdefault("approvals", {})["mode"] = "smart" @@ -63,10 +123,11 @@ PY cp "$REPO_ROOT/agent/SOUL.md" "$HERMES_HOME/SOUL.md" echo " SOUL.md installed" -# Scope to a read-only sandbox: disable host-affecting toolsets. ( cd "$AGENT_DIR" && UV_NO_CONFIG=1 "$VENV_PY" -m hermes_cli.main tools disable \ terminal file code_execution browser computer_use image_gen tts vision web delegation cronjob \ >/dev/null 2>&1 ) || echo " (toolset disable skipped — disable manually with 'hermes tools disable ...')" echo " host toolsets disabled (read-only deepsql + memory/todo/skills remain)" -echo "✓ DeepSQL Agent customization installed. Verify: (cd $AGENT_DIR && uv run hermes mcp test deepsql)" +echo "✓ DeepSQL Agent customization installed." +echo " Verify: (cd $AGENT_DIR && uv run hermes mcp test deepsql)" +echo " Or run: scripts/self-host/setup-agent.sh (starts webui + provisions MCP token)" diff --git a/backend/src/main/java/com/dbaagent/service/AgentChatClient.java b/backend/src/main/java/com/dbaagent/service/AgentChatClient.java index 3bae088..499d2ad 100644 --- a/backend/src/main/java/com/dbaagent/service/AgentChatClient.java +++ b/backend/src/main/java/com/dbaagent/service/AgentChatClient.java @@ -10,6 +10,8 @@ import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.CookieManager; +import java.net.CookiePolicy; import java.net.URI; import java.net.URLEncoder; import java.net.http.HttpClient; @@ -32,12 +34,23 @@ public class AgentChatClient { private static final Logger log = LoggerFactory.getLogger(AgentChatClient.class); + // Cookie jar: the agent scopes sessions by hermes_profile. Without + // /api/profile/switch first, session/new(profile=u-…) then chat/start + // returns 404 "Session not found" under the default profile. + private final CookieManager cookies = new CookieManager(null, CookiePolicy.ACCEPT_ALL); private final HttpClient http = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(5)).build(); + .connectTimeout(Duration.ofSeconds(5)) + .cookieHandler(cookies) + .build(); private final ObjectMapper objectMapper = new ObjectMapper(); - /** Internal base URL of the agent webui (compose network). */ - @Value("${agent.webui-url:http://deepsql-agent:8787}") + /** + * Base URL of the agent webui. Self-host Compose sets + * {@code AGENT_WEBUI_URL=http://host.docker.internal:8787} (no deepsql-agent + * service in the four-container stack). Override for a dedicated agent + * container on the compose network. + */ + @Value("${agent.webui-url:http://host.docker.internal:8787}") private String webuiUrl; /** Hard ceiling on a single agent turn for a channel reply. */ @@ -67,6 +80,13 @@ public String ensureSession(String profile, String existingSessionId) { } public SessionAttempt ensureSessionDetailed(String profile, String existingSessionId) { + try { + switchProfile(profile); + } catch (Exception e) { + log.warn("Could not switch agent profile to {} (webui={}): {}", + profile, webuiUrl, e.toString()); + return new SessionAttempt(null, describe(e)); + } if (existingSessionId != null && !existingSessionId.isBlank()) { return new SessionAttempt(existingSessionId, null); } @@ -101,6 +121,12 @@ public SessionAttempt ensureSessionDetailed(String profile, String existingSessi } } + /** Activate the hermes_profile cookie for subsequent webui API calls. */ + private void switchProfile(String profile) throws Exception { + if (profile == null || profile.isBlank()) return; + postJson("/api/profile/switch", Map.of("name", profile)); + } + /** * Start a turn and block until it finishes (or times out), returning the * assembled assistant text and the tool steps it ran. diff --git a/backend/src/main/java/com/dbaagent/service/scheduler/BrainJobsService.java b/backend/src/main/java/com/dbaagent/service/scheduler/BrainJobsService.java index bc7b951..98f790d 100644 --- a/backend/src/main/java/com/dbaagent/service/scheduler/BrainJobsService.java +++ b/backend/src/main/java/com/dbaagent/service/scheduler/BrainJobsService.java @@ -155,6 +155,12 @@ private BrainJobStatus toStatus(BrainJob job, String connectionId, ScheduledTask statusReason = "Recurring job is not registered in db-scheduler"; } + // db-scheduler leaves consecutive_failures NULL until the first + // success/failure is recorded; coerce before unboxing into the int field. + int consecutiveFailures = row != null && row.consecutiveFailures() != null + ? row.consecutiveFailures() + : 0; + return new BrainJobStatus( job.key(), job.title(), @@ -165,7 +171,7 @@ private BrainJobStatus toStatus(BrainJob job, String connectionId, ScheduledTask row != null ? row.executionTime() : null, row != null ? row.lastSuccess() : null, row != null ? row.lastFailure() : null, - row != null ? row.consecutiveFailures() : 0, + consecutiveFailures, status, statusReason ); diff --git a/backend/src/test/java/com/dbaagent/service/scheduler/BrainJobsServiceTest.java b/backend/src/test/java/com/dbaagent/service/scheduler/BrainJobsServiceTest.java index bf8ccd7..61d4b9e 100644 --- a/backend/src/test/java/com/dbaagent/service/scheduler/BrainJobsServiceTest.java +++ b/backend/src/test/java/com/dbaagent/service/scheduler/BrainJobsServiceTest.java @@ -104,6 +104,38 @@ void listJobs_marksMetadataRefreshRunningAndSchemaDriftActiveByDefault() throws assertEquals("Recurring job is not registered in db-scheduler", driftCheck.statusReason()); } + @Test + void listJobs_toleratesNullConsecutiveFailures() throws Exception { + OffsetDateTime nextRun = OffsetDateTime.of(2026, 4, 8, 9, 30, 0, 0, ZoneOffset.UTC); + + when(schemaChangeTrackingService.ensureDefaultDriftConfig("conn-1")) + .thenReturn(new SchemaDriftConfig()); + when(jdbcTemplate.query(anyString(), org.mockito.ArgumentMatchers.>any())) + .thenAnswer(invocation -> { + @SuppressWarnings("unchecked") + RowMapper mapper = invocation.getArgument(1); + ResultSet rs = mock(ResultSet.class); + when(rs.getString("task_name")).thenReturn("brain-refresh-metadata-lifecycle"); + when(rs.getObject("execution_time", OffsetDateTime.class)).thenReturn(nextRun); + when(rs.getBoolean("picked")).thenReturn(false); + when(rs.getObject("last_success", OffsetDateTime.class)).thenReturn(null); + when(rs.getObject("last_failure", OffsetDateTime.class)).thenReturn(null); + when(rs.getObject("consecutive_failures")).thenReturn(null); + when(rs.getObject("last_heartbeat", OffsetDateTime.class)).thenReturn(null); + return List.of(mapper.mapRow(rs, 0)); + }); + + List jobs = service.listJobs("conn-1"); + + BrainJobsService.BrainJobStatus metadataRefresh = jobs.stream() + .filter(job -> "metadata_refresh".equals(job.key())) + .findFirst() + .orElseThrow(); + + assertEquals("active", metadataRefresh.status()); + assertEquals(0, metadataRefresh.consecutiveFailures()); + } + @Test void runJob_dispatchesMetadataRefreshForConnection() { BrainJobsService.ManualRunResult result = service.runJob("conn-1", "metadata_refresh"); diff --git a/docker-compose.yml b/docker-compose.yml index 461ff46..cb33e5d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -86,10 +86,18 @@ services: # CORS — allow the frontend service (and optional custom domain) cors.allowed.origins: ${CORS_ALLOWED_ORIGINS:-http://localhost:3000} + # Dashboard generation + Slack/channels call Hermes server-side (AgentChatClient), + # not via the browser /agent-api proxy. Point at the host webui; there is no + # deepsql-agent Compose service in the four-service self-host stack. + AGENT_WEBUI_URL: ${AGENT_WEBUI_URL:-http://host.docker.internal:8787} + # Self-host should default to the hardened production profile SPRING_PROFILES_ACTIVE: ${SPRING_PROFILES_ACTIVE:-prod} # Automatically set by install.sh for pgvector self-host mode SPRING_AUTOCONFIGURE_EXCLUDE: ${SPRING_AUTOCONFIGURE_EXCLUDE:-} + # So AGENT_WEBUI_URL=http://host.docker.internal:8787 resolves on Linux too. + extra_hosts: + - "host.docker.internal:host-gateway" ports: - "${DEEPSQL_BACKEND_PORT:-8080}:8080" volumes: @@ -112,6 +120,11 @@ services: depends_on: backend: condition: service_healthy + # nginx proxies /agent-api/ → http://host.docker.internal:8787 (see docker/nginx/default.conf). + # The agent is optional and usually runs on the host (hermes-webui on :8787). + # Docker Desktop injects host.docker.internal; on Linux Compose needs this mapping. + extra_hosts: + - "host.docker.internal:host-gateway" ports: - "${DEEPSQL_FRONTEND_PORT:-3000}:80" diff --git a/docker/nginx/default.conf b/docker/nginx/default.conf index 3151ed0..04334a0 100644 --- a/docker/nginx/default.conf +++ b/docker/nginx/default.conf @@ -71,7 +71,7 @@ server { proxy_set_header Accept "application/json"; } - # Agent chat proxy → the DeepSQL Agent service. + # Agent chat proxy → the DeepSQL Agent service (Hermes webui on :8787). # Mirrors the dev Vite proxy: strip the /agent-api prefix so # /agent-api/api/chat/stream reaches the agent's /api/chat/stream. # SSE buffering MUST be off for token streaming. @@ -79,17 +79,19 @@ server { # Require a valid DeepSQL session before reaching the agent. auth_request /__agent_auth; - # The agent is an optional, separately-run service — the default Compose - # stack does not include it. nginx resolves proxy_pass hostnames at config - # load time, so a literal upstream here makes the whole container refuse to - # start when the agent is absent ("host not found in upstream"). Going - # through a variable defers resolution to request time: without an agent - # this one route returns 502 and everything else keeps working. - resolver 127.0.0.11 ipv6=off valid=10s; - set $deepsql_agent http://deepsql-agent:8787; - proxy_pass $deepsql_agent/; + # The agent is optional and usually runs on the Docker host, not as a + # Compose service. A variable + Docker DNS (127.0.0.11) cannot see + # /etc/hosts entries, so `extra_hosts: deepsql-agent:host-gateway` alone + # still 502s. Literal proxy_pass uses getaddrinfo (hosts file + DNS). + # Compose maps host.docker.internal → host-gateway so this always + # resolves at nginx start; if nothing listens on :8787 this route + # returns 502 and the rest of the UI keeps working. + proxy_pass http://host.docker.internal:8787/; proxy_http_version 1.1; - proxy_set_header Host $host; + # Hermes CSRF compares Origin host:port to Host. `$host` drops the port + # (localhost vs localhost:3000) and profile/switch returns 403 + # "Cross-origin mismatch". `$http_host` preserves the browser Host. + proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/docs/root/SELF_HOST_GUIDE.md b/docs/root/SELF_HOST_GUIDE.md index 19e6b5e..cfc243f 100644 --- a/docs/root/SELF_HOST_GUIDE.md +++ b/docs/root/SELF_HOST_GUIDE.md @@ -142,6 +142,9 @@ the [first-run admin bootstrap](#first-run-access) correctly. In order it: index exists. Any of those missing is a hard failure. 8. Flips `SECURITY_ADMIN_BOOTSTRAP_ENABLED=true`, creates the admin account, then flips it back to `false` in `.env` and restarts the backend. +9. Runs [`scripts/self-host/setup-agent.sh`](../../scripts/self-host/setup-agent.sh) unless + `DEEPSQL_SKIP_AGENT_SETUP=1` — installs Hermes under `~/.hermes/`, provisions the admin + MCP profile, and starts the webui on `0.0.0.0:8787`. Re-running it is the supported way to apply configuration changes and to rebuild after a `git pull`. It is idempotent: already-set secrets are left alone, and the admin bootstrap @@ -250,6 +253,44 @@ rather than trust it. | `SECURITY_SESSION_REFRESH_DAYS` | `application.properties:24` → [`AuthSessionService.java`](../../backend/src/main/java/com/dbaagent/service/AuthSessionService.java) | Refresh-token lifetime. Default 7. | | `SPRING_PROFILES_ACTIVE` | [`docker-compose.yml:90`](../../docker-compose.yml) | `prod` for self-hosting. Compose and `install.sh` both default to it. | +### DeepSQL Agent (Hermes) — required for Agent tab + AI dashboards + +The four Compose services give you auth, brain, classic chat, and schema tools. Two UI +surfaces additionally need a **host Hermes webui** on `:8787`: + +| UI surface | How it reaches Hermes | +|---|---| +| **Agent** tab | Browser → nginx `/agent-api/` → `host.docker.internal:8787` | +| **Dashboards** → AI generate | Backend `AgentChatClient` → `AGENT_WEBUI_URL` (default `http://host.docker.internal:8787`) | + +```bash +./scripts/self-host/setup-agent.sh +``` + +That script (also invoked by `install.sh`) will: + +1. Clone [NousResearch/hermes-agent](https://github.com/NousResearch/hermes-agent) and + [nesquena/hermes-webui](https://github.com/nesquena/hermes-webui) into `~/.hermes/` if needed. +2. Ensure the Python `mcp` SDK is installed (without it, DeepSQL tools never register). +3. Run [`agent/install.sh`](../../agent/install.sh) to write the DBA persona, skills, and + model config from your `DEEPSQL_CHAT_*` values. +4. Mint an MCP token for the admin user and write `~/.hermes/profiles/u-/`. +5. Start the webui bound to `0.0.0.0:8787` with **no** `HERMES_WEBUI_PASSWORD`. + +Compose already sets `AGENT_WEBUI_URL` and `extra_hosts` on the backend, and nginx proxies +`/agent-api/` with `$http_host` (port-preserving) so Hermes CSRF accepts +`Origin: http://localhost:3000`. + +Verify: + +```bash +curl -fsS http://127.0.0.1:8787/api/mcp/servers +./scripts/self-host/smoke-test.sh # includes agent path checks when Hermes is up +``` + +Set `DEEPSQL_SKIP_AGENT_SETUP=1` before `install.sh` if you only want the core stack; +set `DEEPSQL_SMOKE_AGENT=0` to skip agent checks in the smoke test. + ### The LLM [`LlmConfigResolver`](../../backend/src/main/java/com/dbaagent/llm/LlmConfigResolver.java) @@ -525,8 +566,13 @@ not a ping. Against the vault database itself it: verifies the `vector` and `pg_stat_statements` extensions, the `rag_documents` table and its ANN index; logs in as the admin; creates a PostgreSQL connection pointing at the internal `postgres` service; confirms it appears in the connection list; fetches live schema metadata; waits for brain -initialisation to reach `COMPLETED` (up to 20 minutes by default); and finally asserts -that pgvector actually holds embedded documents for that connection. +initialisation to reach `COMPLETED` (up to 20 minutes by default); asserts that pgvector +holds embedded documents for that connection; then (unless `DEEPSQL_SMOKE_AGENT=0`) +verifies Hermes is up, nginx `/agent-api/api/profile/switch` returns 200, and the backend +can create a Hermes session via `AGENT_WEBUI_URL`. + +For a live LLM turn (Agent tab SQL + dashboard HTML), run +[`e2e-agent-check.py`](../../scripts/self-host/e2e-agent-check.py). It reads credentials from `DEEPSQL_INITIAL_ADMIN_EMAIL` / `DEEPSQL_INITIAL_ADMIN_PASSWORD` in `.env`, or from `DEEPSQL_SMOKE_EMAIL` / `DEEPSQL_SMOKE_PASSWORD`. Set @@ -602,30 +648,71 @@ with it. Back up first. ## Troubleshooting -### Frontend container will not start: `host not found in upstream "deepsql-agent"` +### Dashboard generate: 502 “DeepSQL agent is unavailable” +Dashboards call Hermes **from the backend container** (`AgentChatClient`), not through +the browser `/agent-api` proxy. Compose sets `AGENT_WEBUI_URL=http://host.docker.internal:8787` +and `extra_hosts: host.docker.internal:host-gateway` on the backend. If you still see +502, confirm Hermes is up (`curl -sS http://127.0.0.1:8787/api/mcp/servers`) and that +the backend can reach it: + +```bash +docker exec deepsql-selfhost-backend-1 curl -sS -o /dev/null -w '%{http_code}\n' \ + http://host.docker.internal:8787/api/mcp/servers ``` -nginx: [emerg] host not found in upstream "deepsql-agent" in /etc/nginx/conf.d/default.conf:82 -``` -[`docker/nginx/default.conf`](../../docker/nginx/default.conf) contains an `/agent-api/` -location that proxies to `http://deepsql-agent:8787/`, for a separate agent runtime that -**is not one of the four Compose services**. nginx resolves upstream hostnames when it -loads its configuration, so if nothing on the Compose network answers to `deepsql-agent`, -nginx refuses to start and the container crash-loops — `install.sh` then times out waiting -for the frontend. +### `/agent-api/*` returns 502 Bad Gateway + +[`docker/nginx/default.conf`](../../docker/nginx/default.conf) proxies `/agent-api/` to +`http://host.docker.internal:8787/`. The agent is **not** one of the four Compose services — +it is an optional Hermes webui process, typically started on the host at `:8787`. + +The frontend service maps `host.docker.internal` → `host-gateway` via `extra_hosts` in +[`docker-compose.yml`](../../docker-compose.yml). If you still see 502: + +1. Confirm the agent is listening: + `curl -sS -X POST http://127.0.0.1:8787/api/session/new -H 'Content-Type: application/json' -d '{}'` +2. Rebuild the frontend so the nginx config is picked up: + `docker compose -p deepsql-selfhost up -d --build frontend` + +### Agent tab spins / never talks to brain + +Two host-side requirements after Hermes is up on `:8787`: + +1. **Profile cookie.** Hermes scopes sessions by `hermes_profile`. The Agent tab must + `POST /agent-api/api/profile/switch` with `{ "name": "u-" }` before + `session/new` / `chat/start`. Without it, `chat/start` returns 404 + `"Session not found"` and the UI loader never resolves. Fixed in + [`src/lib/api/agentClient.js`](../../src/lib/api/agentClient.js). +2. **Python MCP SDK in the webui process.** Discovery runs in-process. Start the webui + with the agent `venv` that has the `mcp` package (not a bare `.venv` missing it): + + ```bash + # Prefer the canonical venv (has mcp). Bind 0.0.0.0 so Docker nginx can reach it. + export HERMES_HOME=~/.hermes HERMES_WEBUI_HOST=0.0.0.0 HERMES_WEBUI_PORT=8787 + unset HERMES_WEBUI_PASSWORD + cd ~/.hermes/hermes-webui && ~/.hermes/hermes-agent/venv/bin/python server.py + ``` -Confirm it with `docker compose -p deepsql-selfhost logs frontend`. If that is what you -see, remove the `location = /__agent_auth` and `location /agent-api/ { … }` blocks from -`docker/nginx/default.conf` and rebuild the frontend: + Sanity check: `curl -sS http://127.0.0.1:8787/api/mcp/servers` should list + `deepsql` and, after the first agent turn, `active: true` with a non-zero + `tool_count`. If discovery logs `mcp package not installed`, install it into + that interpreter (`uv pip install --python …/venv/bin/python mcp`). + +### Frontend container will not start: `host not found in upstream "…"` + +``` +nginx: [emerg] host not found in upstream "…" in /etc/nginx/conf.d/default.conf +``` + +Ensure `extra_hosts: ["host.docker.internal:host-gateway"]` is present on the frontend +service (Compose adds this for Linux; Docker Desktop usually injects the name already), +then rebuild: ```bash docker compose -p deepsql-selfhost up -d --build frontend ``` -Everything the web UI needs — the SPA and the `/api/` proxy — is unaffected; only the -agent-runtime passthrough goes away. - ### Backend restarts every few minutes during brain init Exit code 0, no shutdown logs, and the restart cadence roughly matching how long brain @@ -701,6 +788,14 @@ Project-name mismatch. See [Project name](#project-name-two-stacks-by-accident). ## Production checklist +Agent-specific (in addition to the core stack): + +- [ ] `./scripts/self-host/setup-agent.sh` succeeded (or was run by `install.sh`) +- [ ] `curl -fsS http://127.0.0.1:8787/api/mcp/servers` returns the `deepsql` server +- [ ] Backend reaches Hermes: `AGENT_WEBUI_URL` + `extra_hosts` on the backend service +- [ ] `./scripts/self-host/smoke-test.sh` passes with agent checks (default `DEEPSQL_SMOKE_AGENT=1`) +- [ ] Optional full turn: `python3 scripts/self-host/e2e-agent-check.py ` + - [ ] `SPRING_PROFILES_ACTIVE=prod` - [ ] `SECURITY_AUTH_ENABLED` left at `true` - [ ] `SECURITY_ADMIN_BOOTSTRAP_ENABLED=false` after the first admin exists diff --git a/scripts/self-host/e2e-agent-check.py b/scripts/self-host/e2e-agent-check.py new file mode 100755 index 0000000..e97f6ec --- /dev/null +++ b/scripts/self-host/e2e-agent-check.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""End-to-end checks for Agent tab + dashboard generate paths. + +Requires a running self-host stack, Hermes on :8787, and admin creds in .env. +Usage (from repo root): + python3 scripts/self-host/e2e-agent-check.py [connectionId] +""" +from __future__ import annotations + +import json +import os +import sys +import time +import urllib.error +import urllib.request +from http.cookiejar import CookieJar +from pathlib import Path +from urllib.request import HTTPCookieProcessor, build_opener + +ROOT = Path(__file__).resolve().parents[2] +ENV = ROOT / ".env" + + +def load_env(path: Path) -> dict[str, str]: + out: dict[str, str] = {} + if not path.exists(): + return out + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + out[k.strip()] = v.strip().strip('"').strip("'") + return out + + +def main() -> int: + env = {**load_env(ENV), **os.environ} + email = env.get("DEEPSQL_INITIAL_ADMIN_EMAIL") or env.get("DEEPSQL_SMOKE_EMAIL") + password = env.get("DEEPSQL_INITIAL_ADMIN_PASSWORD") or env.get("DEEPSQL_SMOKE_PASSWORD") + if not email or not password: + print("Missing admin email/password in .env", file=sys.stderr) + return 1 + + frontend = f"http://localhost:{env.get('DEEPSQL_FRONTEND_PORT', '3000')}" + backend = f"http://localhost:{env.get('DEEPSQL_BACKEND_PORT', '8080')}/api" + conn = sys.argv[1] if len(sys.argv) > 1 else None + + opener = build_opener(HTTPCookieProcessor(CookieJar())) + + def req(url: str, data=None, *, origin: str | None = None, timeout: int = 180): + body = None + headers: dict[str, str] = {} + if data is not None: + body = json.dumps(data).encode() + headers["Content-Type"] = "application/json" + if origin: + headers["Origin"] = origin + headers["Referer"] = origin.rstrip("/") + "/" + r = urllib.request.Request( + url, data=body, headers=headers, method="POST" if data is not None else "GET" + ) + with opener.open(r, timeout=timeout) as resp: + raw = resp.read().decode() or "null" + return json.loads(raw) + + print("→ login") + req(f"{backend}/auth/login", {"email": email, "password": password}) + + if not conn: + conns = req(f"{backend}/connections") + if isinstance(conns, list) and conns: + conn = conns[0].get("connectionId") or conns[0].get("id") + elif isinstance(conns, dict): + items = conns.get("connections") or conns.get("items") or [] + if items: + conn = items[0].get("connectionId") or items[0].get("id") + if not conn: + print("No connectionId available", file=sys.stderr) + return 1 + print(f"→ connection {conn}") + + # ── Agent tab ────────────────────────────────────────────────────────── + print("\n=== Agent tab (browser → /agent-api → Hermes → MCP) ===") + bridge = req(f"{backend}/agent/session", {"connectionId": conn}) + profile = bridge["profile"] + print("profile", profile) + sw = req( + f"{frontend}/agent-api/api/profile/switch", + {"name": profile}, + origin=frontend, + ) + print("switch active", sw.get("active")) + sess = req( + f"{frontend}/agent-api/api/session/new", + {"profile": profile, "enabled_toolsets": ["deepsql", "skills"]}, + origin=frontend, + ) + sid = sess["session"]["session_id"] + print("session", sid) + try: + req( + f"{frontend}/agent-api/api/session/yolo", + {"session_id": sid, "enabled": True}, + origin=frontend, + ) + except Exception as e: + print("yolo (non-fatal)", e) + + msg = ( + f"[Active DeepSQL connection: id {conn}. Use this connection.]\n\n" + "Call mcp_deepsql_execute_sql with SELECT current_database() AS db_name. " + "Reply with just the database name." + ) + start = req( + f"{frontend}/agent-api/api/chat/start", + {"session_id": sid, "message": msg}, + origin=frontend, + ) + stream_id = start["stream_id"] + print("stream", stream_id) + + tokens: list[str] = [] + tools: list[str] = [] + done = False + r = urllib.request.Request( + f"{frontend}/agent-api/api/chat/stream?stream_id={stream_id}", + headers={"Accept": "text/event-stream"}, + ) + with opener.open(r, timeout=300) as resp: + buf = "" + deadline = time.time() + 300 + while time.time() < deadline and not done: + chunk = resp.read(1024) + if not chunk: + break + buf += chunk.decode("utf-8", "replace") + while "\n\n" in buf: + event, buf = buf.split("\n\n", 1) + et, data = "message", "" + for line in event.splitlines(): + if line.startswith("event:"): + et = line[6:].strip() + elif line.startswith("data:"): + data += line[5:].lstrip() + if et in ("stream_end", "done"): + done = True + elif et == "token": + try: + tokens.append(json.loads(data).get("text", "")) + except Exception: + pass + elif et == "tool": + try: + name = json.loads(data).get("name", "") + tools.append(name) + print("TOOL", name) + except Exception: + pass + + answer = "".join(tokens).strip() + print("ANSWER", answer[:500]) + print("TOOLS", tools) + agent_ok = ("dba_agent" in answer.lower()) or any("execute_sql" in t for t in tools) + print("AGENT_OK", agent_ok) + + # ── Dashboard generate ───────────────────────────────────────────────── + print("\n=== Dashboard generate (backend → Hermes) ===") + dash_ok = False + try: + dash = req( + f"{backend}/dashboards/generate", + { + "connectionId": conn, + "prompt": ( + "Create a minimal self-contained HTML dashboard with an h1 " + "'Table Count' and one metric from " + "SELECT count(*)::int AS n FROM information_schema.tables " + "WHERE table_schema = 'public'. Load the dashboard-design skill. " + "Return ONE ```html document only." + ), + }, + timeout=420, + ) + html = "" + if isinstance(dash, dict): + html = dash.get("html") or "" + cfg = dash.get("dashboardConfig") or dash.get("config") or {} + if not html and isinstance(cfg, dict): + html = cfg.get("html") or "" + if not html and dash.get("renderMode") == "artifact": + html = dash.get("html") or "" + if isinstance(dash, dict) and not html: + cfg = dash.get("dashboardConfig") or {} + if isinstance(cfg, dict): + html = cfg.get("html") or "" + print("DASH_KEYS", list(dash.keys())[:15] if isinstance(dash, dict) else type(dash)) + print("HTML_LEN", len(html) if isinstance(html, str) else 0) + dash_ok = isinstance(html, str) and len(html) > 50 and "&2 + echo " and AI dashboards need: ./scripts/self-host/setup-agent.sh" >&2 + fi + echo + fi +else + echo "Skipped agent setup (DEEPSQL_SKIP_AGENT_SETUP=1)." + echo "Run ./scripts/self-host/setup-agent.sh when you want the Agent tab / AI dashboards." + echo +fi + echo "Useful commands:" echo " ./scripts/self-host/status.sh" echo " ./scripts/self-host/smoke-test.sh" +echo " ./scripts/self-host/setup-agent.sh # Hermes webui + MCP profile" +echo " python3 scripts/self-host/e2e-agent-check.py # live Agent+dashboard turn" echo " ./scripts/self-host/uninstall.sh" diff --git a/scripts/self-host/setup-agent.sh b/scripts/self-host/setup-agent.sh new file mode 100755 index 0000000..eeffd54 --- /dev/null +++ b/scripts/self-host/setup-agent.sh @@ -0,0 +1,284 @@ +#!/usr/bin/env bash +# Install and start the DeepSQL Agent (Hermes) for self-host. +# +# The four Compose services alone are NOT enough for the Agent tab or AI +# dashboards — those need a host Hermes webui on :8787 with: +# - Python MCP SDK installed in the webui's interpreter +# - DeepSQL MCP wired to localhost:8080 with a per-user MCP token +# - Binding 0.0.0.0 (so Docker nginx/backend can reach it) +# - No HERMES_WEBUI_PASSWORD (DeepSQL's /agent-api proxy has no Hermes password) +# +# Idempotent. Safe to re-run after `git pull` or credential rotation. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" +ENV_FILE="${DEEPSQL_ENV_FILE:-$ROOT_DIR/.env}" +# Never inherit a nested profile home from a prior Hermes turn +# (e.g. HERMES_HOME=~/.hermes/profiles/u-admin) — that nests clones/config. +if [[ "${HERMES_HOME:-}" == */profiles/* ]]; then + unset HERMES_HOME +fi +HERMES_HOME="${DEEPSQL_HERMES_HOME:-${HERMES_HOME:-$HOME/.hermes}}" +AGENT_DIR="${HERMES_AGENT_DIR:-$HERMES_HOME/hermes-agent}" +WEBUI_DIR="${HERMES_WEBUI_DIR:-$HERMES_HOME/hermes-webui}" +AGENT_REPO="${HERMES_AGENT_REPO:-https://github.com/NousResearch/hermes-agent.git}" +WEBUI_REPO="${HERMES_WEBUI_REPO:-https://github.com/nesquena/hermes-webui.git}" +WEBUI_PORT="${HERMES_WEBUI_PORT:-8787}" +WEBUI_HOST="${HERMES_WEBUI_HOST:-0.0.0.0}" +PID_FILE="${HERMES_HOME}/webui.pid" +LOG_FILE="${HERMES_HOME}/logs/webui.log" +BACKEND_PORT="${DEEPSQL_BACKEND_PORT:-8080}" +FRONTEND_PORT="${DEEPSQL_FRONTEND_PORT:-3000}" + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Error: required command '$1' is not installed." >&2 + exit 1 + fi +} + +resolve_venv_python() { + # Prefer the canonical `venv` (webui's lookup order); fall back to `.venv`. + if [[ -x "$AGENT_DIR/venv/bin/python" ]]; then + echo "$AGENT_DIR/venv/bin/python" + elif [[ -x "$AGENT_DIR/.venv/bin/python" ]]; then + echo "$AGENT_DIR/.venv/bin/python" + else + return 1 + fi +} + +ensure_clone() { + local dir="$1" repo="$2" label="$3" + if [[ -d "$dir/.git" ]]; then + echo "✓ $label already present at $dir" + return 0 + fi + echo "→ Cloning $label into $dir" + mkdir -p "$(dirname "$dir")" + git clone --depth 1 "$repo" "$dir" +} + +ensure_agent_venv() { + if resolve_venv_python >/dev/null; then + return 0 + fi + echo "→ Creating Hermes agent venv" + if command -v uv >/dev/null 2>&1; then + ( cd "$AGENT_DIR" && UV_NO_CONFIG=1 uv sync ) + else + python3 -m venv "$AGENT_DIR/venv" + "$AGENT_DIR/venv/bin/pip" install -U pip + if [[ -f "$AGENT_DIR/pyproject.toml" ]]; then + "$AGENT_DIR/venv/bin/pip" install -e "$AGENT_DIR" + fi + fi + resolve_venv_python >/dev/null || { + echo "Error: could not create a Hermes agent venv under $AGENT_DIR" >&2 + exit 1 + } +} + +ensure_mcp_sdk() { + local py + py="$(resolve_venv_python)" + if "$py" -c "from tools.mcp_tool import _MCP_AVAILABLE; import sys; sys.exit(0 if _MCP_AVAILABLE else 1)" \ + 2>/dev/null; then + echo "✓ Python MCP SDK available ($py)" + return 0 + fi + echo "→ Installing Python MCP SDK into $py" + if command -v uv >/dev/null 2>&1; then + UV_NO_CONFIG=1 uv pip install --python "$py" 'mcp>=1.0' + else + "$py" -m pip install 'mcp>=1.0' + fi + "$py" -c "from tools.mcp_tool import _MCP_AVAILABLE; import sys; sys.exit(0 if _MCP_AVAILABLE else 1)" || { + echo "Error: MCP SDK still unavailable after install. Agent tools will not load." >&2 + exit 1 + } +} + +wait_for_http() { + local url="$1" label="$2" retries="${3:-60}" delay="${4:-2}" + for ((i=1; i<=retries; i++)); do + if curl -fsS "$url" >/dev/null 2>&1; then + echo "✓ $label is healthy: $url" + return 0 + fi + sleep "$delay" + done + echo "Error: timed out waiting for $label at $url" >&2 + return 1 +} + +provision_user_profile() { + # Mint an MCP token for the admin and write ~/.hermes/profiles/u-/ + # so dashboard generation + Agent tab can call DeepSQL MCP as that user. + # Replaces the missing compose provisioner (deepsql-agent:8788). + if [[ ! -f "$ENV_FILE" ]]; then + echo "Warning: no $ENV_FILE — skipping per-user profile provisioning." >&2 + return 0 + fi + # shellcheck disable=SC1090 + set -a; source "$ENV_FILE"; set +a + local email="${DEEPSQL_INITIAL_ADMIN_EMAIL:-}" + local password="${DEEPSQL_INITIAL_ADMIN_PASSWORD:-}" + if [[ -z "$email" || -z "$password" ]]; then + echo "Warning: admin email/password unset — skipping profile provisioning." + echo " After login, re-run this script to write the MCP token into Hermes." + return 0 + fi + + local cookie jar base login_json me_json username profile token_json token + jar="$(mktemp)" + trap 'rm -f "$jar"' RETURN + base="http://127.0.0.1:${BACKEND_PORT}/api" + + if ! wait_for_http "$base/actuator/health" "Backend" 30 2; then + echo "Warning: backend not up — skipping profile provisioning." >&2 + return 0 + fi + + login_json="$(curl -fsS -c "$jar" -H 'Content-Type: application/json' \ + -X POST "$base/auth/login" \ + -d "{\"email\":\"${email}\",\"password\":\"${password}\"}" || true)" + if [[ "$login_json" != *"\"email\""* && "$login_json" != *"\"username\""* ]]; then + echo "Warning: admin login failed — skipping profile provisioning." >&2 + return 0 + fi + + me_json="$(curl -fsS -b "$jar" "$base/auth/me")" + username="$(printf '%s' "$me_json" | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d.get("username") or d.get("name") or "")' 2>/dev/null || true)" + if [[ -z "$username" ]]; then + username="$(printf '%s' "$email" | cut -d@ -f1)" + fi + profile="u-$(printf '%s' "$username" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//')" + + token_json="$(curl -fsS -b "$jar" -H 'Content-Type: application/json' \ + -X POST "$base/auth/mcp-tokens" \ + -d '{"name":"self-host-agent"}')" + token="$(printf '%s' "$token_json" | python3 -c 'import sys,json; print(json.load(sys.stdin).get("token") or "")')" + if [[ -z "$token" ]]; then + echo "Warning: could not mint MCP token — Agent MCP calls will fail until one is configured." >&2 + return 0 + fi + + local profile_home="$HERMES_HOME/profiles/$profile" + mkdir -p "$profile_home" + # Copy root DBA config into the profile, then inject the MCP token. + if [[ -f "$HERMES_HOME/config.yaml" ]]; then + cp "$HERMES_HOME/config.yaml" "$profile_home/config.yaml" + fi + if [[ -f "$HERMES_HOME/SOUL.md" ]]; then + cp "$HERMES_HOME/SOUL.md" "$profile_home/SOUL.md" + fi + + local py + py="$(resolve_venv_python)" + HERMES_HOME="$HERMES_HOME" PROFILE="$profile" TOKEN="$token" REPO_ROOT="$ROOT_DIR" \ + BACKEND_PORT="$BACKEND_PORT" "$py" - <<'PY' +import os, pathlib, yaml +home = pathlib.Path(os.environ["HERMES_HOME"]) / "profiles" / os.environ["PROFILE"] +cfg_path = home / "config.yaml" +cfg = yaml.safe_load(cfg_path.read_text()) if cfg_path.exists() else {} +cfg = cfg or {} +repo = os.environ["REPO_ROOT"] +port = os.environ["BACKEND_PORT"] +token = os.environ["TOKEN"] +cfg.setdefault("mcp_servers", {})["deepsql"] = { + "command": "node", + "args": [f"{repo}/mcp/deepsql-phase1-server.js"], + "env": { + "DEEPSQL_API_BASE_URL": f"http://localhost:{port}/api/", + "DEEPSQL_AUTH_TOKEN": token, + "DEEPSQL_MCP_USER_ID": os.environ["PROFILE"], + "DEEPSQL_MCP_PROJECT_ID": os.environ["PROFILE"], + }, +} +cfg.setdefault("skills", {})["external_dirs"] = [f"{repo}/agent/skills"] +cfg.setdefault("approvals", {})["mode"] = "smart" +cfg_path.write_text(yaml.safe_dump(cfg, sort_keys=False)) +env_path = home / ".env" +env_path.write_text( + f"DEEPSQL_API_BASE_URL=http://localhost:{port}/api/\n" + f"DEEPSQL_AUTH_TOKEN={token}\n" + f"DEEPSQL_MCP_USER_ID={os.environ['PROFILE']}\n" + f"DEEPSQL_MCP_PROJECT_ID={os.environ['PROFILE']}\n" +) +env_path.chmod(0o600) +print(f" profile {os.environ['PROFILE']} written ({cfg_path})") +PY + echo "✓ Provisioned Hermes profile $profile with a fresh MCP token" +} + +start_webui() { + local py + py="$(resolve_venv_python)" + mkdir -p "$(dirname "$LOG_FILE")" "$HERMES_HOME/logs" + + if [[ -f "$PID_FILE" ]] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + echo "✓ Hermes webui already running (pid $(cat "$PID_FILE"))" + return 0 + fi + + # Free a stale listener on the port if our pid file is gone. + if lsof -iTCP:"$WEBUI_PORT" -sTCP:LISTEN >/dev/null 2>&1; then + echo "→ Port $WEBUI_PORT already in use; assuming an existing webui and skipping start." + return 0 + fi + + local certifi + certifi="$("$py" -c 'import certifi; print(certifi.where())' 2>/dev/null || true)" + + echo "→ Starting Hermes webui on ${WEBUI_HOST}:${WEBUI_PORT}" + ( + export HERMES_HOME + export HERMES_WEBUI_HOST="$WEBUI_HOST" + export HERMES_WEBUI_PORT="$WEBUI_PORT" + unset HERMES_WEBUI_PASSWORD + if [[ -n "$certifi" ]]; then + export SSL_CERT_FILE="$certifi" + export REQUESTS_CA_BUNDLE="$certifi" + export CURL_CA_BUNDLE="$certifi" + fi + cd "$WEBUI_DIR" + # Prefer venv/python for the process itself (has MCP when installed there). + nohup "$py" server.py >>"$LOG_FILE" 2>&1 & + echo $! >"$PID_FILE" + ) + wait_for_http "http://127.0.0.1:${WEBUI_PORT}/api/mcp/servers" "Hermes webui" 30 1 +} + +# ── main ──────────────────────────────────────────────────────────────────── +require_command git +require_command curl +require_command node +require_command python3 + +mkdir -p "$HERMES_HOME" +ensure_clone "$AGENT_DIR" "$AGENT_REPO" "hermes-agent" +ensure_clone "$WEBUI_DIR" "$WEBUI_REPO" "hermes-webui" +ensure_agent_venv +ensure_mcp_sdk + +# DBA persona / model / MCP / skills into ~/.hermes +"$ROOT_DIR/agent/install.sh" +# DeepSQL skin on the webui (idempotent) +"$ROOT_DIR/agent/webui/apply-overlay.sh" "$WEBUI_DIR" || true + +provision_user_profile +start_webui + +echo +echo "DeepSQL Agent is ready." +echo " Hermes webui: http://127.0.0.1:${WEBUI_PORT}" +echo " Frontend uses: http://localhost:${FRONTEND_PORT}/agent-api/ → webui" +echo " Backend uses: AGENT_WEBUI_URL=http://host.docker.internal:${WEBUI_PORT}" +echo " Logs: $LOG_FILE" +echo +echo "UI paths that need this process:" +echo " • Agent tab (chat)" +echo " • Dashboards → AI generate" +echo " • Slack (when slack.brain=agent) / CLI deepsql agent" diff --git a/scripts/self-host/smoke-test.sh b/scripts/self-host/smoke-test.sh index 40a28a0..6f5417e 100755 --- a/scripts/self-host/smoke-test.sh +++ b/scripts/self-host/smoke-test.sh @@ -167,7 +167,7 @@ if [[ "$DEEPSQL_SMOKE_WAIT_FOR_INIT" == "true" ]]; then fi fi -if [[ "$VECTOR_STORE_TYPE" == "pgvector" ]]; then +if [[ "$VECTOR_STORE_TYPE" == "pgvector" && "$DEEPSQL_SMOKE_WAIT_FOR_INIT" == "true" ]]; then embedded_docs="$(compose exec -T postgres psql -U postgres -d dba_agent -At -c " SELECT COUNT(embedding) FROM rag_documents @@ -179,5 +179,95 @@ if [[ "$VECTOR_STORE_TYPE" == "pgvector" ]]; then fi fi +# ── Agent paths (Hermes) ──────────────────────────────────────────────────── +# Agent tab (browser→/agent-api) and dashboards (backend→AGENT_WEBUI_URL) both +# need Hermes on :8787. Fail loudly when DEEPSQL_SMOKE_AGENT=1 (default) so a +# "green" smoke test means those UI surfaces will work. +: "${DEEPSQL_SMOKE_AGENT:=1}" +: "${DEEPSQL_FRONTEND_PORT:=3000}" +: "${AGENT_WEBUI_URL:=http://host.docker.internal:8787}" +: "${HERMES_WEBUI_PORT:=8787}" + +if [[ "$DEEPSQL_SMOKE_AGENT" == "1" ]]; then + if ! curl -fsS "http://127.0.0.1:${HERMES_WEBUI_PORT}/api/mcp/servers" >/dev/null 2>&1; then + echo "Error: Hermes webui is not reachable on :${HERMES_WEBUI_PORT}." >&2 + echo " Agent tab and AI dashboards will fail. Run:" >&2 + echo " ./scripts/self-host/setup-agent.sh" >&2 + exit 1 + fi + + # Backend container must reach Hermes (dashboard / Slack / CLI path). + if ! compose exec -T backend sh -c \ + "curl -fsS --connect-timeout 3 \"${AGENT_WEBUI_URL}/api/mcp/servers\" >/dev/null"; then + echo "Error: backend cannot reach AGENT_WEBUI_URL=${AGENT_WEBUI_URL}." >&2 + echo " Check docker-compose.yml AGENT_WEBUI_URL + extra_hosts, and that" >&2 + echo " Hermes binds HERMES_WEBUI_HOST=0.0.0.0." >&2 + exit 1 + fi + + # Browser path through nginx: profile switch must not 403 (Host/Origin CSRF). + switch_code="$(curl -sS -o /tmp/deepsql-agent-switch.json -w '%{http_code}' \ + -b "$cookie_jar" -c "$cookie_jar" \ + -H 'Content-Type: application/json' \ + -H "Origin: http://localhost:${DEEPSQL_FRONTEND_PORT}" \ + -X POST "http://localhost:${DEEPSQL_FRONTEND_PORT}/agent-api/api/profile/switch" \ + -d '{"name":"u-admin"}' || true)" + if [[ "$switch_code" != "200" ]]; then + # Profile name may not be u-admin if the smoke user differs — resolve via bridge. + bridge_json="$(curl -fsS -b "$cookie_jar" -H 'Content-Type: application/json' \ + -X POST "$base/agent/session" -d "{\"connectionId\":\"${connection_id}\"}")" + profile="$(printf '%s' "$bridge_json" | sed -n 's/.*"profile":"\([^"]*\)".*/\1/p')" + if [[ -z "$profile" ]]; then + echo "Error: /api/agent/session did not return a profile." >&2 + echo "$bridge_json" >&2 + exit 1 + fi + switch_code="$(curl -sS -o /tmp/deepsql-agent-switch.json -w '%{http_code}' \ + -b "$cookie_jar" -c "$cookie_jar" \ + -H 'Content-Type: application/json' \ + -H "Origin: http://localhost:${DEEPSQL_FRONTEND_PORT}" \ + -X POST "http://localhost:${DEEPSQL_FRONTEND_PORT}/agent-api/api/profile/switch" \ + -d "{\"name\":\"${profile}\"}" || true)" + else + profile="u-admin" + fi + if [[ "$switch_code" != "200" ]]; then + echo "Error: /agent-api/api/profile/switch → HTTP ${switch_code} (expected 200)." >&2 + echo " Common cause: nginx Host header dropping :${DEEPSQL_FRONTEND_PORT} (CSRF)." >&2 + cat /tmp/deepsql-agent-switch.json 2>/dev/null >&2 || true + exit 1 + fi + + session_json="$(curl -fsS -b "$cookie_jar" -c "$cookie_jar" \ + -H 'Content-Type: application/json' \ + -H "Origin: http://localhost:${DEEPSQL_FRONTEND_PORT}" \ + -X POST "http://localhost:${DEEPSQL_FRONTEND_PORT}/agent-api/api/session/new" \ + -d "{\"profile\":\"${profile}\",\"enabled_toolsets\":[\"deepsql\",\"skills\"]}")" + session_id="$(printf '%s' "$session_json" | python3 -c 'import sys,json; print(json.load(sys.stdin).get("session",{}).get("session_id") or "")' 2>/dev/null || true)" + if [[ -z "$session_id" ]]; then + echo "Error: /agent-api/api/session/new did not return a session_id." >&2 + echo "$session_json" >&2 + exit 1 + fi + + # Backend→Hermes session (dashboard path) — same as AgentChatClient.ensureSession. + backend_switch="$(compose exec -T backend sh -c \ + "curl -fsS -c /tmp/hc.jar -H 'Content-Type: application/json' \ + -X POST '${AGENT_WEBUI_URL}/api/profile/switch' \ + -d '{\"name\":\"${profile}\"}' >/dev/null && \ + curl -fsS -b /tmp/hc.jar -c /tmp/hc.jar -H 'Content-Type: application/json' \ + -X POST '${AGENT_WEBUI_URL}/api/session/new' \ + -d '{\"profile\":\"${profile}\",\"enabled_toolsets\":[\"deepsql\",\"skills\"]}'")" + if [[ "$backend_switch" != *"session_id"* ]]; then + echo "Error: backend→Hermes session/new failed (dashboard path)." >&2 + echo "$backend_switch" >&2 + exit 1 + fi + + echo "Agent smoke checks passed (Hermes up, nginx profile/switch OK, backend session OK)." + echo "Agent profile: $profile" + echo "Agent session: $session_id" +fi + echo "Smoke test passed." echo "Connection ID: ${connection_id}" diff --git a/scripts/self-host/status.sh b/scripts/self-host/status.sh index 253c107..6c3081d 100755 --- a/scripts/self-host/status.sh +++ b/scripts/self-host/status.sh @@ -42,3 +42,18 @@ if curl -fsS "http://localhost:${DEEPSQL_FRONTEND_PORT}" >/dev/null 2>&1; then else echo "unreachable" fi + +: "${HERMES_WEBUI_PORT:=8787}" +: "${AGENT_WEBUI_URL:=http://host.docker.internal:8787}" +printf 'Hermes webui (:%s): ' "$HERMES_WEBUI_PORT" +if curl -fsS "http://127.0.0.1:${HERMES_WEBUI_PORT}/api/mcp/servers" >/dev/null 2>&1; then + echo "ok" +else + echo "unreachable — Agent tab / AI dashboards need ./scripts/self-host/setup-agent.sh" +fi +printf 'Backend → Hermes (%s): ' "$AGENT_WEBUI_URL" +if compose exec -T backend sh -c "curl -fsS --connect-timeout 2 '${AGENT_WEBUI_URL}/api/mcp/servers' >/dev/null" 2>/dev/null; then + echo "ok" +else + echo "unreachable" +fi diff --git a/src/components/AgentChat/AgentChatPanel.jsx b/src/components/AgentChat/AgentChatPanel.jsx index bd70d20..ba140bd 100644 --- a/src/components/AgentChat/AgentChatPanel.jsx +++ b/src/components/AgentChat/AgentChatPanel.jsx @@ -59,6 +59,9 @@ export default function AgentChatPanel({ connectionId, connectionName }) { try { const { profile } = await agentChatAPI.bootstrap(connectionId) profileRef.current = profile + // Hermes requires the hermes_profile cookie before session/chat calls; + // without it, chat/start 404s and the UI loader never resolves. + await agentChatAPI.switchProfile(profile) // Resume the user's most recent conversation for this connection (from our // identity-keyed backend, so it works on any device) unless they asked for diff --git a/src/lib/api/agentClient.js b/src/lib/api/agentClient.js index 9f7fe97..d8670e5 100644 --- a/src/lib/api/agentClient.js +++ b/src/lib/api/agentClient.js @@ -75,6 +75,11 @@ export const agentChatAPI = { return data; }, + /** Re-export for callers that switch explicitly (e.g. AgentChatPanel boot). */ + async switchProfile(profile) { + await switchAgentProfile(profile); + }, + /** Create a lean DBA chat session for this profile; returns the session id. */ async newSession(profile) { // Idempotent re-bind in case bootstrap's switch was skipped or the cookie aged out. @@ -85,6 +90,8 @@ export const agentChatAPI = { } const data = await postJson(`${AGENT_BASE}/api/session/new`, { profile, + // Alias `deepsql` → `mcp-deepsql` once MCP is discovered; `skills` keeps + // the DBA skill surface. Omit host toolsets (terminal/file/etc.). enabled_toolsets: ["deepsql", "skills"], }); const sessionId = data?.session?.session_id || data?.session_id; diff --git a/vite.config.js b/vite.config.js index d839245..8903b27 100644 --- a/vite.config.js +++ b/vite.config.js @@ -56,7 +56,9 @@ export default defineConfig({ // Agent chat service: /agent-api/api/chat/stream → :8787/api/chat/stream '/agent-api': { target: agentProxyTarget, - changeOrigin: true, + // Keep the browser Host (localhost:3000) so Hermes CSRF Origin checks + // pass. changeOrigin:true rewrites Host to :8787 and profile/switch 403s. + changeOrigin: false, rewrite: (p) => p.replace(/^\/agent-api/, ''), timeout: 300000, proxyTimeout: 300000,