From dc8e406ac7fd464ed08b115e27da3f7a91c1a0f1 Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 24 Aug 2026 12:55:35 -0700 Subject: [PATCH 1/9] Run OpenBot on Kubernetes: a Helm chart, and what installing it found One chart for EKS, GKE, AKS and somebody's own cluster, with nothing but values between them. No cloud branching in any template: every place the clouds differ is a value whose default is what a plain self-hosted cluster does. Identity is one annotations map, because that is all IRSA, Workload Identity and AKS workload identity are. Secrets are a plain Secret by default and an ExternalSecret against any backend when asked. Two replicas by default, because horizontal is the point and one hides every bug that is not. A bad install is refused at helm install naming the value to change, rather than found in a crash loop. Three things only a real install could find: drizzle-kit cannot migrate in the shipped image. It reads a TypeScript config, which needs the esbuild that bun install --production leaves out, so it printed one line, exited 1 and said nothing. EMBEDDED_POSTGRES=on was starting containers whose database was never migrated. The migrator inside drizzle-orm is a runtime dependency already and keeps the same journal. sessionOf answered from a map in the process that started the computer, which is right until there are two of them. The replica taking a snapshot is usually not the one handling the click, and an unknown session skips the generation check rather than failing it, so the check that stops a ref from a replaced computer resolving against a live one was silently absent on the shape it was written for. It now asks by listing, never by ensuring, so asking cannot start a computer that had stopped. A browser in an API pod cannot be replicated, so the image's computer gets the same switch its database has. --- .gitignore | 4 + CHANGELOG.md | 40 +++ charts/openbot/.helmignore | 5 + charts/openbot/Chart.yaml | 26 ++ charts/openbot/README.md | 100 +++++++ charts/openbot/ci/aks-values.yaml | 45 +++ charts/openbot/ci/eks-values.yaml | 53 ++++ charts/openbot/ci/gke-values.yaml | 44 +++ charts/openbot/ci/self-hosted-values.yaml | 32 +++ charts/openbot/templates/_helpers.tpl | 264 +++++++++++++++++ charts/openbot/templates/configmap.yaml | 17 ++ charts/openbot/templates/externalsecret.yaml | 24 ++ charts/openbot/templates/httproute.yaml | 26 ++ charts/openbot/templates/ingress.yaml | 44 +++ charts/openbot/templates/migrations/job.yaml | 88 ++++++ charts/openbot/templates/networkpolicy.yaml | 52 ++++ charts/openbot/templates/secret.yaml | 47 ++++ .../openbot/templates/server/deployment.yaml | 120 ++++++++ charts/openbot/templates/server/hpa.yaml | 37 +++ charts/openbot/templates/server/pdb.yaml | 30 ++ charts/openbot/templates/server/service.yaml | 20 ++ .../templates/server/serviceaccount.yaml | 17 ++ charts/openbot/templates/validation.yaml | 107 +++++++ charts/openbot/values.yaml | 265 ++++++++++++++++++ docker/s6/s6-rc.d/computer/run | 13 + docker/s6/scripts/migrate.sh | 5 +- server/scripts/migrate.ts | 43 +++ server/src/computer/supervisor.ts | 41 ++- server/tests/computer-supervisor.test.ts | 76 +++++ 29 files changed, 1679 insertions(+), 6 deletions(-) create mode 100644 charts/openbot/.helmignore create mode 100644 charts/openbot/Chart.yaml create mode 100644 charts/openbot/README.md create mode 100644 charts/openbot/ci/aks-values.yaml create mode 100644 charts/openbot/ci/eks-values.yaml create mode 100644 charts/openbot/ci/gke-values.yaml create mode 100644 charts/openbot/ci/self-hosted-values.yaml create mode 100644 charts/openbot/templates/_helpers.tpl create mode 100644 charts/openbot/templates/configmap.yaml create mode 100644 charts/openbot/templates/externalsecret.yaml create mode 100644 charts/openbot/templates/httproute.yaml create mode 100644 charts/openbot/templates/ingress.yaml create mode 100644 charts/openbot/templates/migrations/job.yaml create mode 100644 charts/openbot/templates/networkpolicy.yaml create mode 100644 charts/openbot/templates/secret.yaml create mode 100644 charts/openbot/templates/server/deployment.yaml create mode 100644 charts/openbot/templates/server/hpa.yaml create mode 100644 charts/openbot/templates/server/pdb.yaml create mode 100644 charts/openbot/templates/server/service.yaml create mode 100644 charts/openbot/templates/server/serviceaccount.yaml create mode 100644 charts/openbot/templates/validation.yaml create mode 100644 charts/openbot/values.yaml create mode 100644 server/scripts/migrate.ts diff --git a/.gitignore b/.gitignore index f5d48e06..01ff5a64 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,7 @@ app/src/lib/generated/application-config.ts # TanStack Router scratch output app/.tanstack/ + +# Helm subchart tarballs, fetched by `helm dependency build`. +charts/*/charts/ +charts/*/Chart.lock diff --git a/CHANGELOG.md b/CHANGELOG.md index cbee9e84..e0ca8557 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,46 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### Run this on Kubernetes + +A Helm chart under `charts/openbot`, and the fixes that installing it for real turned up. + +One chart, four targets: EKS, GKE, AKS and somebody's own cluster, with nothing but values between +them. There is no cloud branching in any template. Every place the clouds genuinely differ is a +value whose default is what a plain self-hosted cluster does: the cluster's own default StorageClass, +no RuntimeClass, a plain Kubernetes Secret, an Ingress. Identity is one `serviceAccount.annotations` +map, which is all IRSA, Workload Identity and AKS workload identity are. Secrets are a plain Secret +by default and an ExternalSecret against any backend when asked, so Secrets Manager, Secret Manager +and Key Vault are a values block rather than three code paths. Gateway API is supported beside +Ingress rather than instead of it. `charts/openbot/ci` holds a values file per target. + +Two replicas by default, because horizontal is the point and one replica hides every bug that is +not. A bad install is refused at `helm install`, naming the value to change, rather than discovered +in a crash loop: no database or two of them, nobody who could sign in, nobody who would be an +administrator, a key of the wrong shape, both routers enabled, or a browser asked for inside more +than one replica. + +**A Bot's computer is not in an API pod.** The image runs one beside the API so that a single +container works on its own, and `EMBEDDED_COMPUTER=off` turns it off. A replica must not carry a +browser: it is a few hundred megabytes holding one Bot's logins, so scaling the API would scale +those with it. + +**Migrations no longer need a development tool.** `bun x drizzle-kit migrate` cannot run in the +shipped image at all. The CLI reads a TypeScript config, which needs the esbuild that +`bun install --production` correctly leaves out, so it printed "Reading config file", exited 1 and +said nothing else. `EMBEDDED_POSTGRES=on` was therefore starting a container whose database was +never migrated, and the first symptom was the API reporting that `users` does not exist. +`server/scripts/migrate.ts` uses the migrator inside `drizzle-orm`, which is a runtime dependency +already, and keeps the same journal, so a database migrated by either tool is migrated. + +**Which run of a computer this is, on more than one replica.** `sessionOf` answered from a map in +the process that started the computer, which is right until there are two: the replica that took a +snapshot is usually not the one handling the click, and the second had nothing to answer with. An +unknown session means "no opinion" and skips the generation check, so on exactly the deployment +shape it was written for, the check that stops a ref from a replaced computer resolving against a +live one was silently absent. It now asks the supervisor when it does not know, by listing rather +than by ensuring, so asking never starts a computer that had stopped. + ### Knowledge searches instead of guessing A package can say which of its skills each coworker gets, and the fintech example gives Knowledge the diff --git a/charts/openbot/.helmignore b/charts/openbot/.helmignore new file mode 100644 index 00000000..8eb5223f --- /dev/null +++ b/charts/openbot/.helmignore @@ -0,0 +1,5 @@ +.DS_Store +.git/ +.gitignore +*.tmproj +ci/ diff --git a/charts/openbot/Chart.yaml b/charts/openbot/Chart.yaml new file mode 100644 index 00000000..625cb19a --- /dev/null +++ b/charts/openbot/Chart.yaml @@ -0,0 +1,26 @@ +apiVersion: v2 +name: openbot +description: Run OpenBot on any Kubernetes cluster, managed or your own +type: application +# The chart's own version, bumped when templates or defaults change. +version: 0.1.0 +# The OpenBot release this chart's default image tag points at. +appVersion: "0.0.4" +home: https://github.com/CopilotKit/OpenBot +sources: + - https://github.com/CopilotKit/OpenBot +keywords: + - openbot + - copilotkit + - agents +maintainers: + - name: CopilotKit + url: https://github.com/CopilotKit +dependencies: + # BUNDLED FOR SOMEBODY TRYING IT, OFF FOR ANYBODY WITH A DATABASE. A deployment with RDS, Cloud SQL + # or Azure Database sets `postgresql.enabled: false` and a connection URL, which is the same line + # the Intelligence chart draws, so somebody who has deployed that does not have to relearn it. + - name: postgresql + version: "~16" + repository: "oci://registry-1.docker.io/bitnamicharts" + condition: postgresql.enabled diff --git a/charts/openbot/README.md b/charts/openbot/README.md new file mode 100644 index 00000000..7fcca671 --- /dev/null +++ b/charts/openbot/README.md @@ -0,0 +1,100 @@ +# OpenBot on Kubernetes + +Runs OpenBot on any Kubernetes cluster: EKS, GKE, AKS, or your own. One chart, four targets, and the +only difference between them is values. + +## Install + +The bundled database and one administrator, which is the shortest thing that works: + +```sh +helm dependency build charts/openbot +helm upgrade --install openbot charts/openbot \ + --namespace openbot --create-namespace \ + --set postgresql.enabled=true \ + --set config.initialAdminEmails=you@example.com \ + --set-string secrets.keyEncryptionKey="$(openssl rand -base64 32)" +``` + +`secrets.keyEncryptionKey` encrypts the credential vault. Generate it once, keep it, and do not put +it in a file anybody commits. The chart marks the Secret it creates `helm.sh/resource-policy: keep`, +so an uninstall does not take the key that every stored credential was encrypted with. + +## What the defaults assume + +**A plain cluster with no cloud features.** The cluster's own default StorageClass, no RuntimeClass, +a plain Kubernetes Secret, an Ingress. There is no cloud branching anywhere in the templates and +there should never be. A deployment on a managed cluster turns things on; a self-hosted one changes +nothing and still works. + +Two replicas by default, because horizontal is the point. Everything that has to survive a replica is +in PostgreSQL, and one replica hides every bug that is not. + +**No browser in the API pod.** The image runs a Bot's computer beside the API so that one container +works on its own. A replica must not carry one: a browser is a few hundred megabytes holding one +Bot's logins, so scaling the API would scale those with it. `server.embeddedComputer` is off here, +and asking for it with more than one replica is refused at install time. + +## Your own database + +```sh +--set postgresql.enabled=false \ +--set database.existingSecret=openbot-database # key: database-url +``` + +Setting both a bundled database and a URL is refused, rather than one of them silently winning. + +## The four targets + +`ci/` holds a values file per target, and each is the shortest thing that expresses what is different +about that cluster: + +| File | What it shows | +| --- | --- | +| `self-hosted-values.yaml` | Nothing turned on. If this file needs to grow, a default is wrong. | +| `eks-values.yaml` | IRSA, Secrets Manager, ALB, zone spread, autoscaling. | +| `gke-values.yaml` | Workload Identity, Secret Manager, Gateway API instead of an Ingress. | +| `aks-values.yaml` | Workload identity, Key Vault, the AKS web app routing class. | + +Render any of them without a cluster: + +```sh +helm template openbot charts/openbot -f charts/openbot/ci/eks-values.yaml +``` + +### Identity, in one map + +IRSA on EKS, Workload Identity on GKE and workload identity on AKS are all annotations on a +ServiceAccount, so `serviceAccount.annotations` covers all three and the chart needs no idea which +cloud it is on. + +### Secrets, without a vendor + +A plain Kubernetes Secret is the default, because that is what a self-hosted cluster has. Setting +`externalSecrets.enabled` turns the same keys into an ExternalSecret against whatever store the +cluster has, so Secrets Manager, Secret Manager and Key Vault are a values block rather than three +code paths. + +### Storage has gravity + +The API tier holds nothing on disk. When per-Bot computers arrive they will, and the ordinary block +volume on all three clouds is **zonal**: once provisioned, every pod referencing it is scheduled into +that zone, so a Bot's computer is pinned to a zone for as long as its profile exists. That is +acceptable and worth stating rather than discovering. `storageClass` stays empty by default, meaning +the cluster's default class, because naming `gp3` or `pd-balanced` here is how a chart stops +installing on somebody's bare-metal cluster. + +## Refused at install, not in a crash loop + +The chart fails the install, naming the value to change, when: there is no database or two of them; +nobody would be an administrator; `singleUser` is combined with a public URL; both an Ingress and an +HTTPRoute are enabled; both `externalSecrets` and an existing Secret are named; or a browser is asked +for inside more than one API replica. + +## Upgrades + +Migrations run as a `pre-install,pre-upgrade` Job, so no replica ever serves in front of a schema it +has not seen. An init container would mean every replica racing to migrate the same database. + +Use `helm upgrade --install --atomic` so a failed upgrade rolls back rather than leaving half a +rollout. diff --git a/charts/openbot/ci/aks-values.yaml b/charts/openbot/ci/aks-values.yaml new file mode 100644 index 00000000..40b8e547 --- /dev/null +++ b/charts/openbot/ci/aks-values.yaml @@ -0,0 +1,45 @@ +# AKS. Azure Database, workload identity, Key Vault through external-secrets. +config: + initialAdminEmails: admin@example.com + intelligence: + apiUrl: https://api.cloud.copilotkit.ai + gatewayWsUrl: wss://gateway.cloud.copilotkit.ai + auth: + google: + clientId: example.apps.googleusercontent.com + publicUrl: https://openbot.example.com +postgresql: + enabled: false +database: + existingSecret: openbot-database +serviceAccount: + annotations: + azure.workload.identity/client-id: 00000000-0000-0000-0000-000000000000 +server: + podLabels: + azure.workload.identity/use: "true" +externalSecrets: + enabled: true + secretStoreRef: + name: azure-key-vault + data: + - secretKey: key-encryption-key + remoteRef: + key: openbot-key-encryption-key + - secretKey: intelligence-api-key + remoteRef: + key: openbot-intelligence-api-key + - secretKey: google-client-secret + remoteRef: + key: openbot/google-client-secret + - secretKey: license-token + remoteRef: + key: openbot-license-token +ingress: + enabled: true + className: webapprouting.kubernetes.azure.com + hosts: + - host: openbot.example.com + paths: + - path: / + pathType: Prefix diff --git a/charts/openbot/ci/eks-values.yaml b/charts/openbot/ci/eks-values.yaml new file mode 100644 index 00000000..08dcbdec --- /dev/null +++ b/charts/openbot/ci/eks-values.yaml @@ -0,0 +1,53 @@ +# EKS. RDS for the database, IRSA for identity, Secrets Manager through external-secrets. +config: + initialAdminEmails: admin@example.com + intelligence: + apiUrl: https://api.cloud.copilotkit.ai + gatewayWsUrl: wss://gateway.cloud.copilotkit.ai + auth: + google: + clientId: example.apps.googleusercontent.com + publicUrl: https://openbot.example.com +postgresql: + enabled: false +database: + existingSecret: openbot-database +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::000000000000:role/openbot +externalSecrets: + enabled: true + secretStoreRef: + name: aws-secrets-manager + data: + - secretKey: key-encryption-key + remoteRef: + key: openbot/key-encryption-key + - secretKey: intelligence-api-key + remoteRef: + key: openbot/intelligence-api-key + - secretKey: google-client-secret + remoteRef: + key: openbot/google-client-secret + - secretKey: license-token + remoteRef: + key: openbot/license-token +ingress: + enabled: true + className: alb + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip +server: + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: openbot + app.kubernetes.io/component: server diff --git a/charts/openbot/ci/gke-values.yaml b/charts/openbot/ci/gke-values.yaml new file mode 100644 index 00000000..66a72850 --- /dev/null +++ b/charts/openbot/ci/gke-values.yaml @@ -0,0 +1,44 @@ +# GKE. Cloud SQL, Workload Identity, Secret Manager through external-secrets, Gateway API. +config: + initialAdminEmails: admin@example.com + intelligence: + apiUrl: https://api.cloud.copilotkit.ai + gatewayWsUrl: wss://gateway.cloud.copilotkit.ai + auth: + google: + clientId: example.apps.googleusercontent.com + publicUrl: https://openbot.example.com +postgresql: + enabled: false +database: + existingSecret: openbot-database +serviceAccount: + annotations: + iam.gke.io/gcp-service-account: openbot@example-project.iam.gserviceaccount.com +externalSecrets: + enabled: true + secretStoreRef: + name: gcp-secret-manager + data: + - secretKey: key-encryption-key + remoteRef: + key: openbot-key-encryption-key + - secretKey: intelligence-api-key + remoteRef: + key: openbot-intelligence-api-key + - secretKey: google-client-secret + remoteRef: + key: openbot/google-client-secret + - secretKey: license-token + remoteRef: + key: openbot-license-token +# Gateway API rather than an Ingress, which the same chart supports without a second code path. +ingress: + enabled: false +httpRoute: + enabled: true + parentRefs: + - name: openbot-gateway + namespace: gateway-system + hostnames: + - openbot.example.com diff --git a/charts/openbot/ci/self-hosted-values.yaml b/charts/openbot/ci/self-hosted-values.yaml new file mode 100644 index 00000000..84720ba3 --- /dev/null +++ b/charts/openbot/ci/self-hosted-values.yaml @@ -0,0 +1,32 @@ +# Somebody's own cluster, which is the shape every default is written for. +# +# Nothing here turns a cloud feature on, because there are none to turn on: the cluster's default +# StorageClass, a plain Secret, the bundled database, an Ingress. If this file needs to grow, a +# default is wrong. +config: + initialAdminEmails: admin@example.com + intelligence: + apiUrl: https://api.cloud.copilotkit.ai + gatewayWsUrl: wss://gateway.cloud.copilotkit.ai + auth: + google: + clientId: example.apps.googleusercontent.com + publicUrl: https://openbot.internal +postgresql: + enabled: true + auth: + # Yours to choose, and the same value on every upgrade. Rendering example only. + password: "example-for-rendering-only" +secrets: + keyEncryptionKey: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + intelligenceApiKey: "example-for-rendering-only" + googleClientSecret: "example-for-rendering-only" + licenseToken: "example-for-rendering-only" +ingress: + enabled: true + className: nginx + hosts: + - host: openbot.internal + paths: + - path: / + pathType: Prefix diff --git a/charts/openbot/templates/_helpers.tpl b/charts/openbot/templates/_helpers.tpl new file mode 100644 index 00000000..7f7a65a8 --- /dev/null +++ b/charts/openbot/templates/_helpers.tpl @@ -0,0 +1,264 @@ +{{/* +Shared shapes, so a component template says what is different about it and nothing else. + +Anything defined here is used by more than one component, or is a decision worth making in exactly +one place. A helper used once belongs in the template that uses it. +*/}} + +{{- define "openbot.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "openbot.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "openbot.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "openbot.labels" -}} +helm.sh/chart: {{ include "openbot.chart" . }} +{{ include "openbot.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end -}} + +{{- define "openbot.selectorLabels" -}} +app.kubernetes.io/name: {{ include "openbot.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* Labels for one component, so two workloads in one release never select each other's pods. */}} +{{- define "openbot.componentLabels" -}} +{{ include "openbot.labels" .root }} +app.kubernetes.io/component: {{ .component }} +{{- end -}} + +{{- define "openbot.componentSelectorLabels" -}} +{{ include "openbot.selectorLabels" .root }} +app.kubernetes.io/component: {{ .component }} +{{- end -}} + +{{- define "openbot.componentName" -}} +{{- printf "%s-%s" (include "openbot.fullname" .root) .component | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "openbot.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} +{{- default (include "openbot.fullname" .) .Values.serviceAccount.name -}} +{{- else -}} +{{- default "default" .Values.serviceAccount.name -}} +{{- end -}} +{{- end -}} + +{{/* The image, with the chart's appVersion as the tag unless one is named. */}} +{{- define "openbot.image" -}} +{{- $tag := default .Chart.AppVersion .Values.image.tag -}} +{{- printf "%s:%s" .Values.image.repository $tag -}} +{{- end -}} + +{{- define "openbot.secretName" -}} +{{- default (printf "%s-secrets" (include "openbot.fullname" .)) .Values.secrets.existingSecret -}} +{{- end -}} + +{{- define "openbot.configMapName" -}} +{{- printf "%s-config" (include "openbot.fullname" .) -}} +{{- end -}} + +{{/* +Where the database is. + +One definition, because the migrations Job and the API must never disagree about it: a Job that +migrated one database while the API talked to another is a failure that looks like a missing table. +*/}} +{{- define "openbot.databaseUrlEnv" -}} +{{- if .Values.postgresql.enabled -}} +{{- /* + THE PASSWORD IS DECLARED FIRST, AND THAT IS NOT A STYLE CHOICE. + + Kubernetes expands `$(VAR)` in an env value only from variables defined earlier in the same list. + Declared after, the reference is left as the literal text `$(POSTGRES_PASSWORD)` and handed to the + server as the password, which fails authentication with `28P01` and reads exactly like a wrong + password rather than like a template that did not expand. +*/}} +- name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ default (printf "%s-postgresql" .Release.Name) .Values.postgresql.auth.existingSecret }} + {{- /* The subchart keeps the superuser's password under its own key, not `password`. */}} + key: {{ eq .Values.postgresql.auth.username "postgres" | ternary "postgres-password" "password" }} +- name: DATABASE_URL + value: postgres://{{ .Values.postgresql.auth.username }}:$(POSTGRES_PASSWORD)@{{ .Release.Name }}-postgresql:5432/{{ .Values.postgresql.auth.database }} +{{- else if .Values.database.existingSecret -}} +- name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ .Values.database.existingSecret }} + key: {{ .Values.database.existingSecretKey }} +{{- else -}} +- name: DATABASE_URL + value: {{ .Values.database.url | quote }} +{{- end -}} +{{- end -}} + +{{/* +Everything the API reads that is not the database. + +Secrets are referenced, never rendered: a value that appears here would appear in `helm get values` +and in whatever holds the release, which is not where `KEY_ENCRYPTION_KEY` belongs. +*/}} +{{- define "openbot.commonEnv" -}} +- name: PORT + value: {{ .Values.server.service.port | quote }} +- name: NODE_ENV + value: production +- name: EMBEDDED_POSTGRES + value: "off" +{{- /* The switch that makes a replica a replica: no browser in an API pod. */}} +- name: EMBEDDED_COMPUTER + value: {{ ternary "on" "off" .Values.server.embeddedComputer | quote }} +- name: TENANT_PACKAGE_DIR + value: {{ .Values.config.tenantPackageDir | quote }} +{{- if .Values.config.publicUrl }} +- name: OPENBOT_PUBLIC_URL + value: {{ .Values.config.publicUrl | quote }} +- name: BETTER_AUTH_URL + value: {{ .Values.config.publicUrl | quote }} +{{- end }} +{{- if .Values.config.initialAdminEmails }} +- name: INITIAL_ADMIN_EMAILS + value: {{ .Values.config.initialAdminEmails | quote }} +{{- end }} +{{- if .Values.config.singleUser }} +- name: OPENBOT_SINGLE_USER + value: "true" +{{- end }} +{{- if .Values.config.logLevel }} +- name: LOG_LEVEL + value: {{ .Values.config.logLevel | quote }} +{{- end }} +{{- if .Values.computers.url }} +- name: AGENT_COMPUTER_URL + value: {{ .Values.computers.url | quote }} +{{- end }} +- name: INTELLIGENCE_API_URL + value: {{ .Values.config.intelligence.apiUrl | quote }} +- name: INTELLIGENCE_GATEWAY_WS_URL + value: {{ .Values.config.intelligence.gatewayWsUrl | quote }} +- name: INTELLIGENCE_API_KEY + valueFrom: + secretKeyRef: + name: {{ include "openbot.secretName" . }} + key: intelligence-api-key +- name: COPILOTKIT_LICENSE_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "openbot.secretName" . }} + key: license-token +{{- with .Values.config.auth.google.clientId }} +- name: GOOGLE_OAUTH_CLIENT_ID + value: {{ . | quote }} +- name: GOOGLE_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "openbot.secretName" $ }} + key: google-client-secret +{{- end }} +{{- with .Values.config.auth.microsoft.clientId }} +- name: MICROSOFT_OAUTH_CLIENT_ID + value: {{ . | quote }} +- name: MICROSOFT_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "openbot.secretName" $ }} + key: microsoft-client-secret +{{- end }} +{{- with .Values.config.auth.microsoft.tenantId }} +- name: MICROSOFT_OAUTH_TENANT_ID + value: {{ . | quote }} +{{- end }} +{{- with .Values.config.auth.okta.clientId }} +- name: OKTA_OAUTH_CLIENT_ID + value: {{ . | quote }} +- name: OKTA_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "openbot.secretName" $ }} + key: okta-client-secret +{{- end }} +{{- with .Values.config.auth.okta.issuer }} +- name: OKTA_OAUTH_ISSUER + value: {{ . | quote }} +{{- end }} +- name: KEY_ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: {{ include "openbot.secretName" . }} + key: key-encryption-key +- name: BETTER_AUTH_SECRET + valueFrom: + secretKeyRef: + name: {{ include "openbot.secretName" . }} + key: better-auth-secret + optional: true +- name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: {{ include "openbot.secretName" . }} + key: model-api-key + optional: true +- name: COMPUTER_TOKEN + valueFrom: + secretKeyRef: + name: {{ default (include "openbot.secretName" .) .Values.computers.existingTokenSecret }} + key: computer-token + optional: true +{{- with .Values.config.extraEnv }} +{{ toYaml . }} +{{- end }} +{{- end -}} + +{{/* +Keeping replicas apart. + +Soft by default, so a one-node cluster still schedules. A deployment that means it sets +`podAntiAffinity: hard` and gets a replica per node, or writes its own `affinity` and gets neither. +*/}} +{{- define "openbot.podAntiAffinity" -}} +{{- $root := .root -}} +{{- $component := .component -}} +{{- if $root.Values.server.affinity -}} +{{ toYaml $root.Values.server.affinity }} +{{- else if eq (default "soft" $root.Values.server.podAntiAffinity) "hard" -}} +podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: +{{ include "openbot.componentSelectorLabels" (dict "root" $root "component" $component) | indent 10 }} +{{- else if eq (default "soft" $root.Values.server.podAntiAffinity) "soft" -}} +podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: +{{ include "openbot.componentSelectorLabels" (dict "root" $root "component" $component) | indent 12 }} +{{- end -}} +{{- end -}} diff --git a/charts/openbot/templates/configmap.yaml b/charts/openbot/templates/configmap.yaml new file mode 100644 index 00000000..11bf220b --- /dev/null +++ b/charts/openbot/templates/configmap.yaml @@ -0,0 +1,17 @@ +{{/* +Non-secret configuration, in a ConfigMap so a change to it rolls the pods. + +Everything here is readable by anybody who can read the namespace, which is the test for whether a +value belongs in this file rather than in the Secret beside it. +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "openbot.configMapName" . }} + labels: +{{ include "openbot.labels" . | indent 4 }} +data: + OPENBOT_DEPLOYMENT: kubernetes + {{- if .Values.config.publicUrl }} + TRUSTED_ORIGINS: {{ .Values.config.publicUrl | quote }} + {{- end }} diff --git a/charts/openbot/templates/externalsecret.yaml b/charts/openbot/templates/externalsecret.yaml new file mode 100644 index 00000000..caebbb87 --- /dev/null +++ b/charts/openbot/templates/externalsecret.yaml @@ -0,0 +1,24 @@ +{{- if .Values.externalSecrets.enabled }} +{{/* +The same keys, from whatever the cluster's secret store is. + +Backend-agnostic on purpose: Secrets Manager, Secret Manager and Key Vault are all a `secretStoreRef` +and a list of remote keys, so none of them appears in this chart by name. +*/}} +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: {{ include "openbot.secretName" . }} + labels: +{{ include "openbot.labels" . | indent 4 }} +spec: + refreshInterval: {{ .Values.externalSecrets.refreshInterval }} + secretStoreRef: + name: {{ required "externalSecrets.secretStoreRef.name is required when externalSecrets.enabled is true" .Values.externalSecrets.secretStoreRef.name }} + kind: {{ .Values.externalSecrets.secretStoreRef.kind }} + target: + name: {{ include "openbot.secretName" . }} + creationPolicy: Owner + data: +{{ toYaml (required "externalSecrets.data must name at least key-encryption-key" .Values.externalSecrets.data) | indent 4 }} +{{- end }} diff --git a/charts/openbot/templates/httproute.yaml b/charts/openbot/templates/httproute.yaml new file mode 100644 index 00000000..fdb9a988 --- /dev/null +++ b/charts/openbot/templates/httproute.yaml @@ -0,0 +1,26 @@ +{{- if .Values.httpRoute.enabled }} +{{/* +Gateway API, as an alternative rather than a replacement. + +Where this is going, and not where every cluster is: plenty of self-hosted clusters still run an +Ingress controller, so both exist here and neither is assumed. Turning both on is a mistake rather +than a merge, and `validation.yaml` says so at install time. +*/}} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "openbot.fullname" . }} + labels: +{{ include "openbot.labels" . | indent 4 }} +spec: + parentRefs: +{{ toYaml (required "httpRoute.parentRefs is required when httpRoute.enabled is true" .Values.httpRoute.parentRefs) | indent 4 }} + {{- with .Values.httpRoute.hostnames }} + hostnames: +{{ toYaml . | indent 4 }} + {{- end }} + rules: + - backendRefs: + - name: {{ include "openbot.fullname" . }}-server + port: {{ .Values.server.service.port }} +{{- end }} diff --git a/charts/openbot/templates/ingress.yaml b/charts/openbot/templates/ingress.yaml new file mode 100644 index 00000000..53224f2e --- /dev/null +++ b/charts/openbot/templates/ingress.yaml @@ -0,0 +1,44 @@ +{{- if .Values.ingress.enabled }} +{{- $fullname := include "openbot.fullname" . -}} +{{- $port := .Values.server.service.port -}} +{{/* +Getting traffic in, one of two ways. + +`className` has no default because the controller differs on every cluster and always will, and the +annotations that configure it differ with it. Naming one here would be a chart that installs on the +cluster it was written on. +*/}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullname }} + labels: +{{ include "openbot.labels" . | indent 4 }} + {{- with .Values.ingress.annotations }} + annotations: +{{ toYaml . | indent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: +{{ toYaml . | indent 4 }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ $fullname }}-server + port: + number: {{ $port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/openbot/templates/migrations/job.yaml b/charts/openbot/templates/migrations/job.yaml new file mode 100644 index 00000000..d6d1783a --- /dev/null +++ b/charts/openbot/templates/migrations/job.yaml @@ -0,0 +1,88 @@ +{{- if .Values.migrations.enabled }} +{{- $component := "migrations" -}} +{{/* +The schema, before anything serves a request. + +A pre-install and pre-upgrade hook rather than an init container on the Deployment: with several +replicas, an init container means every replica races to migrate the same database, and the +migration that loses is a failed pod on an otherwise healthy rollout. One Job runs once. + +WHEN IT RUNS DEPENDS ON WHOSE DATABASE IT IS, and both wrong answers deadlock rather than slow down. + +A pre-install hook runs before the chart's own resources, so with the bundled database the +StatefulSet does not exist yet and never will while the hook is running: waiting cannot help, +because the thing being waited for is behind the wait. A post-install hook has the mirror image of +the same problem, because `--wait` holds it until the Deployment is ready and the Deployment cannot +be ready against a database with no schema in it. + +So on a first install with the bundled database this is not a hook at all. It is an ordinary Job, +created in the same pass as the database and the Deployment, and it waits for the database while the +replicas restart against a schema that is on its way. Nothing is ordered because nothing needs to +be: the Job's own wait is the ordering, and a replica that starts too early is a restart rather than +a failure. + +Every other case is a hook, because in every other case the database already exists. An external +database on install, and any upgrade at all, run `pre`, where the schema is ready before a replica +reaches a version of it that it has not seen. + +Deleted before the next run and on success. A failed one is kept, so the reason is still readable. +*/}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" $component) }}-{{ .Release.Revision }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} + {{- if not (and .Values.postgresql.enabled .Release.IsInstall) }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-5" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + {{- end }} +spec: + activeDeadlineSeconds: {{ .Values.migrations.activeDeadlineSeconds }} + backoffLimit: {{ .Values.migrations.backoffLimit }} + template: + metadata: + labels: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 8 }} + spec: + restartPolicy: Never + serviceAccountName: {{ include "openbot.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: migrate + image: {{ include "openbot.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + workingDir: /app/server + command: + - /bin/sh + - -ec + - | + # Bounded, so a database that is never coming back fails the install rather than + # holding it open until somebody notices. `pg_isready` reads the same DATABASE_URL the + # migration is about to use, so this cannot wait on the wrong server. + deadline=$(( $(date +%s) + {{ .Values.migrations.waitForDatabaseSeconds }} )) + until pg_isready -d "$DATABASE_URL" -q; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "the database did not accept connections within {{ .Values.migrations.waitForDatabaseSeconds }}s" >&2 + exit 1 + fi + sleep 2 + done + exec /usr/local/bin/bun scripts/migrate.ts + env: +{{ include "openbot.databaseUrlEnv" . | indent 12 }} + - name: EMBEDDED_POSTGRES + value: "off" + - name: EMBEDDED_COMPUTER + value: "off" + {{- with .Values.migrations.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} +{{- end }} diff --git a/charts/openbot/templates/networkpolicy.yaml b/charts/openbot/templates/networkpolicy.yaml new file mode 100644 index 00000000..1f21b02f --- /dev/null +++ b/charts/openbot/templates/networkpolicy.yaml @@ -0,0 +1,52 @@ +{{- if .Values.networkPolicy.enabled }} +{{- $component := "server" -}} +{{/* +What the API may reach, and what may reach it. + +Off by default, because a NetworkPolicy on a cluster with no CNI that enforces one is a resource +that silently does nothing, and on a cluster that does enforce one a wrong rule is an outage. A +deployment that turns this on is saying it knows which of the two it has. + +Egress deliberately allows DNS and the database, and nothing else without being asked: a Bot's +computer reaching the open internet is the computers' own policy, not the API's. +*/}} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} +spec: + podSelector: + matchLabels: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 6 }} + policyTypes: + - Ingress + - Egress + ingress: + - ports: + - port: {{ .Values.server.service.port }} + protocol: TCP + {{- with .Values.networkPolicy.extraIngress }} +{{ toYaml . | indent 4 }} + {{- end }} + egress: + # DNS, or nothing resolves and every failure looks like the database being down. + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + {{- if .Values.postgresql.enabled }} + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: postgresql + ports: + - port: 5432 + protocol: TCP + {{- end }} + {{- with .Values.networkPolicy.extraEgress }} +{{ toYaml . | indent 4 }} + {{- end }} +{{- end }} diff --git a/charts/openbot/templates/secret.yaml b/charts/openbot/templates/secret.yaml new file mode 100644 index 00000000..96c084d8 --- /dev/null +++ b/charts/openbot/templates/secret.yaml @@ -0,0 +1,47 @@ +{{- if and (not .Values.secrets.existingSecret) (not .Values.externalSecrets.enabled) }} +{{/* +The fallback, and the default, because a plain Kubernetes Secret is what a self-hosted cluster has. + +A deployment on a cloud points `externalSecrets` at its own store instead and this renders nothing. +Either way the templates that read these keys are identical, which is the point: no vendor appears +anywhere except in a values block. + +`helm.sh/resource-policy: keep` is deliberate. `KEY_ENCRYPTION_KEY` is what the credential vault is +encrypted with, so a `helm uninstall` that took it would leave every stored credential unreadable +even after a reinstall against the same database. +*/}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openbot.secretName" . }} + labels: +{{ include "openbot.labels" . | indent 4 }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +stringData: + key-encryption-key: {{ required "secrets.keyEncryptionKey is required unless secrets.existingSecret or externalSecrets is used. Generate one with: openssl rand -base64 32" .Values.secrets.keyEncryptionKey | quote }} + intelligence-api-key: {{ required "secrets.intelligenceApiKey is required. OpenBot needs CopilotKit Intelligence and refuses to start without it." .Values.secrets.intelligenceApiKey | quote }} + license-token: {{ required "secrets.licenseToken is required. OpenBot needs CopilotKit Intelligence and refuses to start without it." .Values.secrets.licenseToken | quote }} + {{- with .Values.secrets.betterAuthSecret }} + better-auth-secret: {{ . | quote }} + {{- end }} + {{- with .Values.secrets.modelApiKey }} + model-api-key: {{ . | quote }} + {{- end }} + {{- with .Values.secrets.googleClientSecret }} + google-client-secret: {{ . | quote }} + {{- end }} + {{- with .Values.secrets.microsoftClientSecret }} + microsoft-client-secret: {{ . | quote }} + {{- end }} + {{- with .Values.secrets.oktaClientSecret }} + okta-client-secret: {{ . | quote }} + {{- end }} + {{- with .Values.secrets.computerToken }} + computer-token: {{ . | quote }} + {{- end }} + {{- with .Values.secrets.supervisorToken }} + supervisor-token: {{ . | quote }} + {{- end }} +{{- end }} diff --git a/charts/openbot/templates/server/deployment.yaml b/charts/openbot/templates/server/deployment.yaml new file mode 100644 index 00000000..abbe4b6c --- /dev/null +++ b/charts/openbot/templates/server/deployment.yaml @@ -0,0 +1,120 @@ +{{- $component := "server" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} + {{- with .Values.commonAnnotations }} + annotations: +{{ toYaml . | indent 4 }} + {{- end }} +spec: + {{- /* Left unset when the HPA owns it, so a chart upgrade cannot undo a scale the HPA decided. */}} + {{- if not .Values.server.autoscaling.enabled }} + replicas: {{ .Values.server.replicaCount }} + {{- end }} + selector: + matchLabels: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 6 }} + template: + metadata: + labels: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 8 }} + {{- with .Values.server.podLabels }} +{{ toYaml . | indent 8 }} + {{- end }} + annotations: + {{- /* + Roll the pods when configuration changes. + + Without this a `helm upgrade` that only changes the ConfigMap leaves every replica running + the old values, and the deployment looks upgraded while behaving exactly as it did. + */}} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.server.podAnnotations }} +{{ toYaml . | indent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "openbot.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.podSecurityContext }} + securityContext: +{{ toYaml . | indent 8 }} + {{- end }} + {{- $affinity := include "openbot.podAntiAffinity" (dict "root" . "component" $component) }} + {{- if $affinity }} + affinity: +{{ $affinity | indent 8 }} + {{- end }} + {{- with .Values.server.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.server.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.server.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: server + image: {{ include "openbot.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.securityContext }} + securityContext: +{{ toYaml . | indent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.server.service.port }} + protocol: TCP + env: +{{ include "openbot.databaseUrlEnv" . | indent 12 }} +{{ include "openbot.commonEnv" . | indent 12 }} + envFrom: + - configMapRef: + name: {{ include "openbot.configMapName" . }} + {{- with .Values.config.extraEnvFrom }} +{{ toYaml . | indent 12 }} + {{- end }} + {{- /* + Three probes, and the startup one is the reason the other two can be impatient. + + A cold start migrates nothing but does read and validate the tenant package, so first + boot is slower than steady state. Without a startup probe the liveness one has to be + slack enough for the slowest boot, which means a wedged replica stays in the Service for + as long as a healthy slow one would. + */}} + startupProbe: + httpGet: + path: /health + port: http + periodSeconds: {{ .Values.server.startupProbe.periodSeconds }} + failureThreshold: {{ .Values.server.startupProbe.failureThreshold }} + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: {{ .Values.server.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.server.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.server.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.server.readinessProbe.failureThreshold }} + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: {{ .Values.server.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.server.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.server.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.server.livenessProbe.failureThreshold }} + {{- with .Values.server.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} diff --git a/charts/openbot/templates/server/hpa.yaml b/charts/openbot/templates/server/hpa.yaml new file mode 100644 index 00000000..6a79a421 --- /dev/null +++ b/charts/openbot/templates/server/hpa.yaml @@ -0,0 +1,37 @@ +{{- if .Values.server.autoscaling.enabled }} +{{- $component := "server" -}} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + minReplicas: {{ .Values.server.autoscaling.minReplicas }} + maxReplicas: {{ .Values.server.autoscaling.maxReplicas }} + metrics: + {{- if .Values.server.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.server.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.server.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.server.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} + {{- with .Values.server.autoscaling.behavior }} + behavior: +{{ toYaml . | indent 4 }} + {{- end }} +{{- end }} diff --git a/charts/openbot/templates/server/pdb.yaml b/charts/openbot/templates/server/pdb.yaml new file mode 100644 index 00000000..83cd913f --- /dev/null +++ b/charts/openbot/templates/server/pdb.yaml @@ -0,0 +1,30 @@ +{{- if .Values.server.podDisruptionBudget.enabled }} +{{- $component := "server" -}} +{{/* +What a drain may take at once. + +Only meaningful with more than one replica: a budget of `minAvailable: 1` over a single replica means +a node drain blocks forever rather than being safe, so this is rendered only where it can be kept. +*/}} +{{- $replicas := int .Values.server.replicaCount -}} +{{- if .Values.server.autoscaling.enabled }} +{{- $replicas = int .Values.server.autoscaling.minReplicas -}} +{{- end }} +{{- if gt $replicas 1 }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} +spec: + {{- if .Values.server.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.server.podDisruptionBudget.maxUnavailable }} + {{- else }} + minAvailable: {{ .Values.server.podDisruptionBudget.minAvailable }} + {{- end }} + selector: + matchLabels: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/openbot/templates/server/service.yaml b/charts/openbot/templates/server/service.yaml new file mode 100644 index 00000000..b9bcda46 --- /dev/null +++ b/charts/openbot/templates/server/service.yaml @@ -0,0 +1,20 @@ +{{- $component := "server" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} + {{- with .Values.server.service.annotations }} + annotations: +{{ toYaml . | indent 4 }} + {{- end }} +spec: + type: {{ .Values.server.service.type }} + ports: + - port: {{ .Values.server.service.port }} + targetPort: http + protocol: TCP + name: http + selector: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 4 }} diff --git a/charts/openbot/templates/server/serviceaccount.yaml b/charts/openbot/templates/server/serviceaccount.yaml new file mode 100644 index 00000000..a4c19681 --- /dev/null +++ b/charts/openbot/templates/server/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "openbot.serviceAccountName" . }} + labels: +{{ include "openbot.labels" . | indent 4 }} + {{- /* + IRSA, Workload Identity and AKS workload identity are all annotations here, which is why this is + one map and not three code paths. The chart never learns which cloud it is on. + */}} + {{- with .Values.serviceAccount.annotations }} + annotations: +{{ toYaml . | indent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/openbot/templates/validation.yaml b/charts/openbot/templates/validation.yaml new file mode 100644 index 00000000..c25847a2 --- /dev/null +++ b/charts/openbot/templates/validation.yaml @@ -0,0 +1,107 @@ +{{/* +Refused at install time, not discovered in a crash loop. + +Every check here is something the API would fail on at boot, or something that is quietly wrong +rather than loudly broken. A `helm install` that succeeds and leaves pods restarting is worse than +one that refuses and says why, because the second names the value to change. + +This template renders nothing. +*/}} + +{{- if and (not .Values.postgresql.enabled) (not .Values.database.url) (not .Values.database.existingSecret) }} +{{- fail "No database. Either set postgresql.enabled=true to run the bundled one, or set database.url (or database.existingSecret) to point at your own." }} +{{- end }} + +{{- /* + The bundled database, without a password anybody can produce again. + + The subchart generates one on install and then refuses to render on upgrade without being handed + the current value, so a release installed without one cannot be upgraded, only reinstalled. That + failure arrives on the second deploy, which is the worst time to find it. +*/}} +{{- if and .Values.postgresql.enabled (not .Values.postgresql.auth.password) (not .Values.postgresql.auth.existingSecret) }} +{{- fail "The bundled database needs a password you can supply again on upgrade. Set postgresql.auth.password, or postgresql.auth.existingSecret to name one you made. Generate one with: openssl rand -base64 24" }} +{{- end }} + +{{- if and .Values.postgresql.enabled (or .Values.database.url .Values.database.existingSecret) }} +{{- fail "Two databases. postgresql.enabled is true and database.url is also set, so it is not clear which one is meant. Pick one." }} +{{- end }} + +{{- /* + Nobody can get in, which is two separate mistakes rather than one. + + Naming administrators does not create a way to sign in, and configuring sign-in does not make + anybody an administrator: the server refuses to start without a provider, and grants the role to + nobody without the addresses. An earlier version of this check accepted admin emails alone, which + passed the install and then crash-looped on the first boot, so both are checked and named apart. +*/}} +{{- $provider := or .Values.config.auth.google.clientId .Values.config.auth.microsoft.clientId .Values.config.auth.okta.clientId -}} +{{- if and (not .Values.config.singleUser) (not $provider) }} +{{- fail "Nobody could sign in. Configure config.auth.google, config.auth.microsoft or config.auth.okta, or set config.singleUser=true for a local trial where every request is one fixed administrator." }} +{{- end }} + +{{- if and $provider (not .Values.config.initialAdminEmails) }} +{{- fail "Nobody would be an administrator. config.initialAdminEmails is what grants the role; nothing else does." }} +{{- end }} + +{{- if and $provider (not .Values.config.publicUrl) }} +{{- fail "Sign-in needs somewhere to come back to. Set config.publicUrl to the address people reach this deployment at, which is where the OAuth callback lands." }} +{{- end }} + +{{- if and .Values.config.auth.okta.clientId (not .Values.config.auth.okta.issuer) }} +{{- fail "config.auth.okta.issuer is required alongside the Okta client, such as https://example.okta.com/oauth2/default. It is what makes it a particular Okta rather than Okta in general." }} +{{- end }} + +{{- if and .Values.config.singleUser .Values.config.publicUrl }} +{{- fail "config.singleUser serves every visitor as one administrator, so it must not be combined with a public URL. Configure an identity provider and set config.initialAdminEmails instead." }} +{{- end }} + +{{- /* + Intelligence, which is not optional. + + All four values are required together and the server refuses to start on a partial set, so the + same rule is applied here: caught at install with the values named, rather than in a crash loop + whose message is in a log nobody has opened. +*/}} +{{- if or (not .Values.config.intelligence.apiUrl) (not .Values.config.intelligence.gatewayWsUrl) }} +{{- fail "OpenBot requires CopilotKit Intelligence. Set config.intelligence.apiUrl and config.intelligence.gatewayWsUrl, and the matching secrets.intelligenceApiKey and secrets.licenseToken." }} +{{- end }} + +{{- if and .Values.ingress.enabled .Values.httpRoute.enabled }} +{{- fail "ingress.enabled and httpRoute.enabled are both set. They are two ways to do the same thing; pick the one your cluster runs." }} +{{- end }} + +{{- /* + A key of the wrong shape. + + `KEY_ENCRYPTION_KEY` must decode to exactly 32 bytes, and the server refuses to start otherwise. + Checked here only when the chart is the one creating the Secret: a key coming from an existing + Secret or an external store is not readable at template time, and guessing about it would mean + refusing installs that are fine. +*/}} +{{- if and .Values.secrets.keyEncryptionKey (not .Values.secrets.existingSecret) (not .Values.externalSecrets.enabled) }} +{{- if ne (len (b64dec .Values.secrets.keyEncryptionKey)) 32 }} +{{- fail "secrets.keyEncryptionKey must be a base64-encoded 32-byte value. Generate one with: openssl rand -base64 32" }} +{{- end }} +{{- end }} + +{{- if and .Values.externalSecrets.enabled .Values.secrets.existingSecret }} +{{- fail "externalSecrets.enabled and secrets.existingSecret are both set. The first creates the Secret and the second says one already exists; pick one." }} +{{- end }} + +{{- /* + A replica count that cannot survive the thing replicas are for. + + One replica is a supported way to run this and not the default, so this is a note rather than a + refusal only when somebody has also asked for a disruption budget, where one replica means a node + drain blocks rather than being safe. +*/}} +{{- if and (not .Values.server.autoscaling.enabled) (lt (int .Values.server.replicaCount) 1) }} +{{- fail "server.replicaCount must be at least 1." }} +{{- end }} + +{{- if .Values.server.embeddedComputer }} +{{- if gt (int .Values.server.replicaCount) 1 }} +{{- fail "server.embeddedComputer runs a browser inside every API pod, which cannot be replicated: each replica would hold a profile directory belonging to one Bot. Either set server.replicaCount=1, or leave embeddedComputer off and give the Bots a computer of their own." }} +{{- end }} +{{- end }} diff --git a/charts/openbot/values.yaml b/charts/openbot/values.yaml new file mode 100644 index 00000000..dc9107d5 --- /dev/null +++ b/charts/openbot/values.yaml @@ -0,0 +1,265 @@ +# OpenBot, on any Kubernetes cluster. +# +# THE DEFAULTS ARE WHAT A PLAIN CLUSTER CAN DO. Every place the clouds genuinely differ is a value, +# and its default is whatever a self-hosted cluster with no cloud features does: the cluster's own +# default StorageClass, no RuntimeClass, a plain Kubernetes Secret, bundled PostgreSQL, an Ingress. +# A deployment on EKS, GKE or AKS turns things on; a self-hosted one changes nothing and still works. +# There is no cloud branching anywhere in the templates, and there should never be: a chart that only +# installs cleanly on a managed cluster has failed the thing this repository is for. + +nameOverride: "" +fullnameOverride: "" + +image: + repository: ghcr.io/copilotkit/openbot + # Empty means the chart's appVersion, so an upgrade of the chart moves the image with it. + tag: "" + pullPolicy: IfNotPresent +imagePullSecrets: [] + +server: + replicaCount: 2 + # Two by default because horizontal is the point. Everything that has to survive a replica is in + # PostgreSQL, and one replica hides every bug that is not. + + # THE BROWSER IS NOT IN THIS POD. The image runs a Bot's computer beside the API for the + # one-container case; a replica of the API must not carry one, because a browser is a few hundred + # megabytes holding one Bot's logins, and scaling the API would scale those with it. + embeddedComputer: false + + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + memory: 2Gi + + # Off by default. Turn it on and set the metric you actually want to scale on. + autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: null + behavior: {} + + podDisruptionBudget: + enabled: true + minAvailable: 1 + maxUnavailable: null + + # Spread replicas so a node or zone going away does not take the deployment with it. Soft by + # default: a single-node cluster still schedules, which is what a first `helm install` runs on. + topologySpreadConstraints: [] + podAntiAffinity: soft + + nodeSelector: {} + tolerations: [] + affinity: {} + podAnnotations: {} + podLabels: {} + + service: + type: ClusterIP + port: 3001 + annotations: {} + + # Probes hit the API's own health route. + livenessProbe: + initialDelaySeconds: 20 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + startupProbe: + periodSeconds: 5 + failureThreshold: 60 + +serviceAccount: + create: true + name: "" + # ONE MAP, THREE CLOUDS. IRSA on EKS, Workload Identity on GKE and on AKS are all annotations on a + # ServiceAccount, so this covers every one of them and the chart needs no idea which it is on. + # eks.amazonaws.com/role-arn: arn:aws:iam:::role/ + # iam.gke.io/gcp-service-account: @.iam.gserviceaccount.com + # azure.workload.identity/client-id: + annotations: {} + automountServiceAccountToken: false + +# The database migrations, run as a Job before the API starts. A pre-upgrade hook, so a rollout +# never puts a new replica in front of a schema it has not seen. +migrations: + enabled: true + backoffLimit: 3 + # How long to wait for the database to accept connections before giving up. A pre-install hook runs + # before the chart's own resources, so on a first install with the bundled database this Job starts + # before the database does. + waitForDatabaseSeconds: 300 + # A ceiling on the whole Job, so a migration that hangs fails the release rather than holding it. + activeDeadlineSeconds: 900 + # Kept on failure so somebody can read why. Helm deletes it before the next run. + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi + +# What OpenBot needs to run. Anything secret belongs in `secrets` below, never here. +config: + # Where people reach this deployment. Sets the OAuth callback and the trusted origin. + publicUrl: "" + # Administrators by email address, comma-separated. Required unless singleUser is on. + initialAdminEmails: "" + # Every request is one fixed administrator. Local trials only; the deployment refuses to start + # with no identity provider unless this says somebody meant it. + singleUser: false + + # How people sign in. One of these, or `singleUser` above, and nothing else will do: the server + # refuses to start rather than serve a deployment where every visitor is an administrator. + # + # Client secrets go under `secrets` below. Only the public halves belong here. + auth: + google: + clientId: "" + microsoft: + clientId: "" + # `common` is Microsoft's own default and admits personal accounts as well as work ones. A + # company that means "our staff" puts its directory GUID here. + tenantId: "" + okta: + clientId: "" + issuer: "" + tenantPackageDir: /app/examples/fintech + + # CopilotKit Intelligence, which OpenBot requires. All four values are needed together: the server + # refuses to start on a partial set, deliberately, because a half-configured Intelligence is a + # mistake somebody made rather than a deployment that meant to run without one. + intelligence: + apiUrl: "" + gatewayWsUrl: "" + # The two secret halves live under `secrets` below, never here. + logLevel: "" + # Free-form additions, for anything this chart has no opinion about. + extraEnv: [] + extraEnvFrom: [] + +# How a Bot gets a computer. `shared` is one browser for every Bot, which is the only shape this +# slice of the chart ships; per-Bot computers arrive with the Sandbox provider. +computers: + mode: shared + url: "" + # Never a literal. See `secrets` below. + existingTokenSecret: "" + +database: + # Used when `postgresql.enabled` is false. A URL, or a secret holding one. + url: "" + existingSecret: "" + existingSecretKey: "database-url" + +# The bundled database. Off by default in favour of a managed one. +postgresql: + enabled: false + auth: + # THE SUPERUSER, DELIBERATELY, AND ONLY FOR THIS BUNDLED DATABASE. + # + # The first migration runs `CREATE EXTENSION vector` and a later one drops it again, and only a + # superuser can do either. An ordinary role fails the install at migration one with "permission + # denied to create extension", which reads like a broken chart rather than a database privilege. + # + # This is the shape `docker-compose.yml` already ships, where the app user is the cluster + # superuser, and it is safe for the same reason: this database exists for this release alone and + # nothing else can reach it. + # + # A MANAGED DATABASE IS NOT THIS. On RDS, Cloud SQL or Azure there is no superuser to hand out, + # so create the extension once as the administrative role and grant the migrating role ownership + # of it. `CREATE EXTENSION IF NOT EXISTS` then passes for an ordinary user. + username: postgres + database: openbot + # SET ONE, OR UPGRADES FAIL. The subchart generates a password on install and then refuses to + # render on upgrade unless it is given the current one, which turns the second `helm upgrade` + # into an error about credentials. A password you chose, or a Secret you made, makes the release + # repeatable. The chart refuses to install without one rather than letting you find out later. + password: "" + existingSecret: "" + image: + repository: bitnamilegacy/postgresql + primary: + persistence: + enabled: true + # EMPTY MEANS THE CLUSTER'S DEFAULT CLASS, and that is deliberate. Naming `gp3` or + # `pd-balanced` here is exactly how a chart stops installing on somebody's bare-metal cluster. + storageClass: "" + size: 20Gi + +# Secrets, without picking a vendor. +# +# A plain Kubernetes Secret is the default because that is what a self-hosted cluster has. +# `externalSecrets` turns the same fields into an ExternalSecret instead, so Secrets Manager, Secret +# Manager and Key Vault are a values block rather than three code paths. +secrets: + # Point at a Secret you made yourself, and the chart creates none. + existingSecret: "" + # Created by the chart when `existingSecret` is empty. Pass with --set-string or a values file that + # is not in version control; `KEY_ENCRYPTION_KEY` and the model credential are the two that matter + # and neither should ever be a literal in a file anybody commits. + keyEncryptionKey: "" + betterAuthSecret: "" + modelApiKey: "" + computerToken: "" + supervisorToken: "" + intelligenceApiKey: "" + licenseToken: "" + # The client secret for whichever provider is configured above. + googleClientSecret: "" + microsoftClientSecret: "" + oktaClientSecret: "" + +externalSecrets: + enabled: false + # Whatever your cluster's ClusterSecretStore or SecretStore is called. Backend-agnostic on purpose. + secretStoreRef: + name: "" + kind: ClusterSecretStore + refreshInterval: 1h + # remoteRef keys, one per secret this chart reads. + data: [] + +ingress: + enabled: false + # The controller differs on every cluster and always will, so this is a value with no default. + className: "" + annotations: {} + hosts: + - host: openbot.example.com + paths: + - path: / + pathType: Prefix + tls: [] + +# Gateway API, as an alternative rather than a replacement. Plenty of self-hosted clusters still run +# an Ingress controller, so both are here and neither is assumed. +httpRoute: + enabled: false + parentRefs: [] + hostnames: [] + +networkPolicy: + enabled: false + # Where the API may reach out to. A deployment with a managed database adds its CIDR here. + extraEgress: [] + extraIngress: [] + +podSecurityContext: + runAsNonRoot: false + fsGroup: null +securityContext: {} + +# Applied to every pod this chart creates. +commonLabels: {} +commonAnnotations: {} diff --git a/docker/s6/s6-rc.d/computer/run b/docker/s6/s6-rc.d/computer/run index 77989bda..d671a813 100755 --- a/docker/s6/s6-rc.d/computer/run +++ b/docker/s6/s6-rc.d/computer/run @@ -1,8 +1,21 @@ #!/command/with-contenv sh # The Bot's browser. Bound to loopback: its only caller is the API beside it. # +# `EMBEDDED_COMPUTER=on` is the default, because one container that just works is what this image is +# for. Off, this service exits 0 immediately and s6 leaves it alone, the same way `postgres` does, +# and the API reaches a computer elsewhere through `AGENT_COMPUTER_URL` or a supervisor. +# +# That switch is what lets this image be a stateless replica. A browser is a few hundred megabytes of +# memory holding one Bot's logins, so an API pod carrying one is neither stateless nor cheap to run +# several of: scaling the API would scale the browsers with it, and every replica would hold a +# profile directory that matters to exactly one Bot. +# # `with-contenv` is not decoration. Without it s6 starts a service with none of the container's # environment, and the failure is a config error naming a variable that is plainly set. +set -eu +if [ "${EMBEDDED_COMPUTER:-on}" != "on" ]; then + exec /bin/true +fi cd /app/agent-computer export PORT=4100 export WORKSPACE_DIR=/workspace diff --git a/docker/s6/scripts/migrate.sh b/docker/s6/scripts/migrate.sh index 3a71d4df..6776b1e2 100755 --- a/docker/s6/scripts/migrate.sh +++ b/docker/s6/scripts/migrate.sh @@ -8,4 +8,7 @@ set -eu [ "${EMBEDDED_POSTGRES:-off}" = "on" ] || exit 0 cd /app/server -exec s6-setuidgid pwuser /usr/local/bin/bun x drizzle-kit migrate --config=drizzle.config.ts +# `scripts/migrate.ts`, not `drizzle-kit`. The CLI is a development dependency and needs esbuild to +# read its TypeScript config, which `bun install --production` leaves out of this image: asked to +# migrate here it exits 1 without printing why, and the container comes up against an empty database. +exec s6-setuidgid pwuser /usr/local/bin/bun scripts/migrate.ts diff --git a/server/scripts/migrate.ts b/server/scripts/migrate.ts new file mode 100644 index 00000000..e4dddbed --- /dev/null +++ b/server/scripts/migrate.ts @@ -0,0 +1,43 @@ +/** + * Apply the migrations, using only what a running deployment already has. + * + * NOT `drizzle-kit migrate`, and that is the whole point of this file. The CLI is a development + * dependency: it reads `drizzle.config.ts`, which means compiling TypeScript, which means the esbuild + * that `bun install --production` correctly leaves out of a runtime image. Asked to migrate there it + * prints "Reading config file", exits 1, and says nothing at all, so a deployment looks like it + * migrated and comes up against an empty database complaining that `users` does not exist. + * + * The migrator underneath it is part of `drizzle-orm`, which is a runtime dependency because the + * server imports it anyway. It needs a connection and the folder of SQL files, both of which are in + * the image, and it keeps the same `drizzle.__drizzle_migrations` journal the CLI does, so the two + * are interchangeable and a database migrated by either is migrated. + */ +import { join } from "node:path"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { migrate } from "drizzle-orm/postgres-js/migrator"; +import postgres from "postgres"; + +const databaseUrl = process.env.DATABASE_URL; +if (!databaseUrl) { + throw new Error( + "DATABASE_URL must be configured before running a database migration command", + ); +} + +/* + * One connection, and `max: 1`. + * + * Migrations are a single ordered conversation with the database, and a pool would let two + * statements that must be ordered land on different connections. + */ +const client = postgres(databaseUrl, { max: 1, onnotice: () => {} }); + +try { + await migrate(drizzle(client), { + migrationsFolder: join(import.meta.dir, "..", "drizzle"), + }); + console.info(JSON.stringify({ type: "migrations-applied", status: "ok" })); +} finally { + // Released whatever happened, so a failure exits rather than hanging on an open socket. + await client.end({ timeout: 5 }); +} diff --git a/server/src/computer/supervisor.ts b/server/src/computer/supervisor.ts index 37c388f7..76ac0201 100644 --- a/server/src/computer/supervisor.ts +++ b/server/src/computer/supervisor.ts @@ -13,14 +13,22 @@ * honest about being one shared computer. */ -import type { ComputerStatus } from "./schema"; import type { ComputerLocation, ComputerProvider } from "./provider"; +import type { ComputerStatus } from "./schema"; /** * The last container start time seen for each Bot, from the `/ensure` that located it. * * Not a cache in front of the supervisor: `locate` still calls it every time. This only carries the * answer the few lines to whoever needs to know which run of the computer they are talking to. + * + * PROCESS-LOCAL, AND THEREFORE NEVER THE ONLY ANSWER. On one replica the snapshot and the click that + * follows it are the same process, so this is always populated by the time anything asks. On several + * they are usually not, and a replica that has never located this Bot has nothing here. `resolve` + * reads an unknown session as "no opinion" and skips the generation check, so an empty map does not + * fail — it silently stops checking, on exactly the deployment shape the check was written for. So + * `sessionOf` falls back to asking, and this stays what it always was: a way to skip the round trip + * on the replica that just did the work. */ const sessions = new Map(); @@ -157,11 +165,34 @@ export function createDockerSupervisorProvider( async sessionOf(botId: string): Promise { /* * Read from the same `/ensure` every action already makes, and remembered rather than asked - * for again: `locate` runs immediately before the call that needs this, so the value is as - * fresh as the address it was fetched with. Asking twice would double the supervisor's work on - * the hot path to learn something it just told us. + * for again: on the replica that located this Bot, `locate` ran immediately before the call + * that needs this, so the value is as fresh as the address it was fetched with. Asking twice + * would double the supervisor's work on the hot path to learn something it just told us. */ - return sessions.get(botId); + const known = sessions.get(botId); + if (known) return known; + + /* + * Nothing here means another replica did the work, not that there is nothing to know. + * + * LISTING, NOT ENSURING, and the difference is the whole feature. `/ensure` starts a computer + * that is not running, so answering "which run is this" with it would wake every idle Bot that + * anything asked about, and a deployment that suspends idle computers would quietly never + * suspend one. Listing is a read: a Bot with no computer answers undefined, which is the same + * answer as before and leaves the check exactly where it was. + */ + try { + const computers = await listRaw(); + const startedAt = computers.find( + (computer) => computer.botId === botId, + )?.startedAt; + if (startedAt) sessions.set(botId, startedAt); + return startedAt; + } catch { + // Unknown, not mismatched. A supervisor that cannot be reached must not turn every ref into + // a refusal; the generation check goes back to being skipped, which is where it started. + return undefined; + } }, async locate(botId: string): Promise { diff --git a/server/tests/computer-supervisor.test.ts b/server/tests/computer-supervisor.test.ts index 919d4cc3..ddc38ced 100644 --- a/server/tests/computer-supervisor.test.ts +++ b/server/tests/computer-supervisor.test.ts @@ -236,3 +236,79 @@ describe("Docker supervisor provider", () => { expect(await provider.reset("bot")).toEqual({ cleared: false }); }); }); + +/** + * Which run of a computer this is, asked by a replica that did not start it. + * + * `sessionOf` is what stops a ref from a dead container resolving against a live one: a replaced + * computer counts generations from one again, so the generation alone cannot tell them apart. It + * answers from what the last `/ensure` reported, which is free and correct while one process does + * both halves of the work. + * + * On more than one replica it is neither. The replica that took the snapshot is very often not the + * replica handling the click, and the second one has never called `/ensure` for that Bot, so it has + * nothing to answer with. `resolve` treats an unknown session as "no opinion" and skips the check by + * design, which is right for a provider that cannot tell and wrong here: the check is simply absent, + * silently, on exactly the deployment shape it was written for. + */ +describe("telling one run of a computer from the next, across replicas", () => { + /* + * A bot id of its own per test, because the map this reads is module scope. + * + * Two providers in one process are not two replicas: they share it. A test that located the + * computer under one name and then asked under the same name would pass whatever the code did, + * which is the shape of a test that proves nothing. + */ + const startedAt = "2026-08-24T09:00:00.000Z"; + + function replica(botId: string, seen: string[] = []) { + const running = [ + { + botId, + container: `openbot-computer-${botId}`, + status: "running", + url: `http://openbot-computer-${botId}:4100`, + startedAt, + }, + ]; + return createDockerSupervisorProvider({ + baseUrl: "http://supervisor:4300", + token: "t", + fetchImpl: (async (url: string | URL | Request) => { + const path = new URL(String(url)).pathname; + seen.push(path); + if (path.endsWith("/ensure")) return Response.json(running[0]); + return Response.json({ computers: running }); + }) as unknown as typeof fetch, + }); + } + + test("a replica that located the computer knows the run", async () => { + const client = replica("located"); + await client.locate("located"); + expect(await client.sessionOf?.("located")).toBe(startedAt); + }); + + test("a replica that never located it still knows the run", async () => { + /* + * The regression. This replica is serving the click; another one took the snapshot. Without an + * answer here the generation check is skipped and a ref from a computer that has since been + * replaced resolves against the new one, which is the case the check exists for. + */ + const client = replica("never-located"); + expect(await client.sessionOf?.("never-located")).toBe(startedAt); + }); + + test("asking does not start a computer that is not running", async () => { + /* + * The other half, and the easier one to get wrong. `/ensure` starts a computer; answering this + * question with it would mean every idle Bot is woken by being asked about, which is how a + * deployment ends up never suspending anything and never noticing, because everything works and + * only the bill says otherwise. + */ + const seen: string[] = []; + const client = replica("asked-about", seen); + await client.sessionOf?.("asked-about"); + expect(seen.some((path) => path.endsWith("/ensure"))).toBe(false); + }); +}); From 7619bcece88aab3a0a5af6cb88168e9eedaac1d0 Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 24 Aug 2026 13:28:41 -0700 Subject: [PATCH 2/9] Give Bots computers on Kubernetes, and suspend them when idle The chart had no computer, so no Bot could do anything on a cluster. It has one now, and a Bot has driven a real browser on real EKS with the decision in the audit trail. computers.mode picks the shape. shared runs one browser for every Bot and needs nothing installed. sandbox gives each Bot its own as a Sandbox from kubernetes-sigs/agent-sandbox, which is built for exactly this: an isolated stateful singleton with a stable identity and persistent storage, where suspending is a field that keeps the volumes, so a computer comes back with its logins rather than signed out. What decides a computer is idle is the audit trail, not the browser. Asking the browser wakes it, so every computer anything asked about would come back up and the bill would never fall. The work is claimed and leased out of Postgres with for update skip locked. Three features need that one mechanism, so it is written once with all three in view: the culler here, routines, and a hop from one Bot to another. A CronJob runs the sweep rather than a timer in the API, because a timer fires in every replica and suspending a browser somebody just started using is not something to do five times. Also: a fresh EKS cluster very often has no default StorageClass. eksctl creates gp2, unmarked and on the in-tree provisioner current Kubernetes no longer has, so a volume asking for the default never binds and nothing says why. Found on a real 1.34 cluster and written down where somebody configuring one will read it. --- CHANGELOG.md | 20 +- charts/openbot/README.md | 46 + charts/openbot/ci/aks-values.yaml | 6 + charts/openbot/ci/eks-values.yaml | 26 + charts/openbot/ci/gke-values.yaml | 6 + charts/openbot/ci/self-hosted-values.yaml | 4 + charts/openbot/templates/_helpers.tpl | 20 +- .../templates/computer/culler-cronjob.yaml | 59 + .../templates/computer/sandbox-rbac.yaml | 46 + .../templates/computer/sandbox-template.yaml | 105 + .../openbot/templates/computer/service.yaml | 19 + .../templates/computer/statefulset.yaml | 161 ++ .../openbot/templates/computer/warmpool.yaml | 22 + charts/openbot/templates/validation.yaml | 20 + charts/openbot/values.yaml | 57 +- server/drizzle.config.ts | 1 + server/drizzle/0016_durable_work.sql | 14 + server/drizzle/meta/0016_snapshot.json | 2482 +++++++++++++++++ server/drizzle/meta/_journal.json | 7 + server/scripts/cull-idle-computers.ts | 62 + server/src/computer/provider.ts | 64 +- server/src/computer/sandbox.ts | 318 +++ server/src/config.ts | 70 +- server/src/db/schema/index.ts | 3 +- server/src/db/schema/work.ts | 79 + server/src/work/culler.ts | 174 ++ server/src/work/queue.ts | 180 ++ .../tests/computer-culler.integration.test.ts | 208 ++ server/tests/work-queue.integration.test.ts | 187 ++ 29 files changed, 4456 insertions(+), 10 deletions(-) create mode 100644 charts/openbot/templates/computer/culler-cronjob.yaml create mode 100644 charts/openbot/templates/computer/sandbox-rbac.yaml create mode 100644 charts/openbot/templates/computer/sandbox-template.yaml create mode 100644 charts/openbot/templates/computer/service.yaml create mode 100644 charts/openbot/templates/computer/statefulset.yaml create mode 100644 charts/openbot/templates/computer/warmpool.yaml create mode 100644 server/drizzle/0016_durable_work.sql create mode 100644 server/drizzle/meta/0016_snapshot.json create mode 100644 server/scripts/cull-idle-computers.ts create mode 100644 server/src/computer/sandbox.ts create mode 100644 server/src/db/schema/work.ts create mode 100644 server/src/work/culler.ts create mode 100644 server/src/work/queue.ts create mode 100644 server/tests/computer-culler.integration.test.ts create mode 100644 server/tests/work-queue.integration.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e0ca8557..e60fab07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,9 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ### Run this on Kubernetes -A Helm chart under `charts/openbot`, and the fixes that installing it for real turned up. +A Helm chart under `charts/openbot`, Bots and all, and the fixes that installing it for real turned +up. Proven on a real EKS cluster: five workloads, replicas across two nodes, EBS volumes bound, and a +Bot opening a real page from inside AWS with the decision in the audit trail. One chart, four targets: EKS, GKE, AKS and somebody's own cluster, with nothing but values between them. There is no cloud branching in any template. Every place the clouds genuinely differ is a @@ -40,6 +42,22 @@ never migrated, and the first symptom was the API reporting that `users` does no `server/scripts/migrate.ts` uses the migrator inside `drizzle-orm`, which is a runtime dependency already, and keeps the same journal, so a database migrated by either tool is migrated. +**A computer for each Bot, suspended when idle.** `computers.mode: sandbox` gives every Bot its own +browser as a `Sandbox` from `kubernetes-sigs/agent-sandbox`, which is built for this workload: an +isolated, stateful, singleton pod with a stable identity and persistent storage. Suspending is one +field, and it keeps the volumes, so a computer comes back with its logins rather than signed out of +everything. `shared` stays the default and needs nothing installed in the cluster. + +**What decides a computer is idle is the audit trail, not the browser.** Asking the browser would +wake it, so every computer anything asked about would come back up and the bill would never fall. + +**Durable work, claimed by whichever replica gets there first.** `work_items` plus +`select ... for update skip locked` and a lease: no coordinator, no leader election, and a replica +added is throughput added. The idle-computer culler is its first user; scheduled routines and +hand-offs between Bots are the other two, which is why it is written once rather than three times +slightly differently. A CronJob runs the sweep, because a timer in the API fires in every replica and +suspending a browser somebody just started using is not something to do five times. + **Which run of a computer this is, on more than one replica.** `sessionOf` answered from a map in the process that started the computer, which is right until there are two: the replica that took a snapshot is usually not the one handling the click, and the second had nothing to answer with. An diff --git a/charts/openbot/README.md b/charts/openbot/README.md index 7fcca671..89a837ec 100644 --- a/charts/openbot/README.md +++ b/charts/openbot/README.md @@ -75,6 +75,25 @@ A plain Kubernetes Secret is the default, because that is what a self-hosted clu cluster has, so Secrets Manager, Secret Manager and Key Vault are a values block rather than three code paths. +### Check for a default StorageClass first + +A fresh EKS cluster very often has none. `eksctl` creates `gp2`, which is not marked default and uses +the in-tree `kubernetes.io/aws-ebs` provisioner that current Kubernetes no longer has. A volume asking +for "the default" then never binds, the computer sits `Pending`, and nothing says why. One line tells +you: + +```sh +kubectl get sc +``` + +Either create a default class backed by `ebs.csi.aws.com`, or set `computers.persistence.storageClass` +and `postgresql.primary.persistence.storageClass` to one that exists. `ci/eks-values.yaml` does the +second. + +`volumeBindingMode: WaitForFirstConsumer` matters on every cloud: without it the volume is created in +a zone chosen before the pod is scheduled, and pods stick unschedulable with a node-affinity conflict. +That only happens in multi-zone clusters, so it passes every single-zone test. + ### Storage has gravity The API tier holds nothing on disk. When per-Bot computers arrive they will, and the ordinary block @@ -91,6 +110,33 @@ nobody would be an administrator; `singleUser` is combined with a public URL; bo HTTPRoute are enabled; both `externalSecrets` and an existing Secret are named; or a browser is asked for inside more than one API replica. +## A computer for each Bot + +`computers.mode` decides how a Bot gets a browser: + +| Mode | What it does | Needs | +| --- | --- | --- | +| `shared` | One browser for every Bot, run by this chart. | Nothing. | +| `sandbox` | A computer each, suspended when idle and resumed with its logins intact. | The `agent-sandbox` controller in the cluster. | +| `external` | Neither; `computers.url` points at one somebody else runs. | Nothing. | + +`shared` is what a first install should use. Sessions, files and logins are shared between Bots in +that mode, which is stated on the fleet page rather than hidden. + +`sandbox` uses `kubernetes-sigs/agent-sandbox`, whose `Sandbox` CRD is built for exactly this +workload: an isolated, stateful, singleton pod with a stable identity and persistent storage. +Suspending is `operatingMode: Suspended`, which terminates the pod and keeps the volumes. + +**What decides that a computer is idle** is the audit trail, not the browser. Asking the browser +would wake it, so every computer anything asked about would come back up and the bill would never +fall. That is the known, invisible way to lose scale-to-zero: everything works, nothing suspends. + +**A CronJob does the suspending, not a timer in the API.** Every replica would fire its own timer and +each would decide independently to suspend the same computer. The work is claimed and leased out of +PostgreSQL with `select ... for update skip locked`, so whichever pod runs the sweep takes what +nobody else holds, and one that dies mid-suspend hands its work back when the lease expires. The +decision is re-checked at the moment of acting, because somebody may have come back in between. + ## Upgrades Migrations run as a `pre-install,pre-upgrade` Job, so no replica ever serves in front of a schema it diff --git a/charts/openbot/ci/aks-values.yaml b/charts/openbot/ci/aks-values.yaml index 40b8e547..363d9226 100644 --- a/charts/openbot/ci/aks-values.yaml +++ b/charts/openbot/ci/aks-values.yaml @@ -32,6 +32,9 @@ externalSecrets: - secretKey: google-client-secret remoteRef: key: openbot/google-client-secret + - secretKey: computer-token + remoteRef: + key: openbot/computer-token - secretKey: license-token remoteRef: key: openbot-license-token @@ -43,3 +46,6 @@ ingress: paths: - path: / pathType: Prefix + +computers: + mode: shared diff --git a/charts/openbot/ci/eks-values.yaml b/charts/openbot/ci/eks-values.yaml index 08dcbdec..4374aaf0 100644 --- a/charts/openbot/ci/eks-values.yaml +++ b/charts/openbot/ci/eks-values.yaml @@ -1,4 +1,20 @@ # EKS. RDS for the database, IRSA for identity, Secrets Manager through external-secrets. +# +# CHECK THE CLUSTER HAS A DEFAULT STORAGECLASS BEFORE INSTALLING, because a fresh EKS cluster very +# often does not. `eksctl` creates `gp2`, which is not marked default and uses the in-tree +# `kubernetes.io/aws-ebs` provisioner that no longer exists in current Kubernetes. A volume asking +# for "the default" then never binds, the computer sits Pending, and nothing says why. Verified on a +# real 1.34 cluster; `kubectl get sc` shows it in one line. +# +# Either create a default class: +# +# provisioner: ebs.csi.aws.com, volumeBindingMode: WaitForFirstConsumer, +# annotated storageclass.kubernetes.io/is-default-class: "true" +# +# or name one here, which is what the line below does. `WaitForFirstConsumer` is not optional on any +# cloud: without it the volume is created in a zone picked before the pod is scheduled, and pods stick +# unschedulable with a node-affinity conflict, in multi-zone clusters only, so it passes every +# single-zone test. config: initialAdminEmails: admin@example.com intelligence: @@ -29,6 +45,9 @@ externalSecrets: - secretKey: google-client-secret remoteRef: key: openbot/google-client-secret + - secretKey: computer-token + remoteRef: + key: openbot/computer-token - secretKey: license-token remoteRef: key: openbot/license-token @@ -51,3 +70,10 @@ server: matchLabels: app.kubernetes.io/name: openbot app.kubernetes.io/component: server + +computers: + mode: shared + persistence: + # Named here rather than in the chart, which is the whole point of a per-target values file: the + # chart must not name `gp3`, and a deployment on EKS should. + storageClass: gp3 diff --git a/charts/openbot/ci/gke-values.yaml b/charts/openbot/ci/gke-values.yaml index 66a72850..ada42429 100644 --- a/charts/openbot/ci/gke-values.yaml +++ b/charts/openbot/ci/gke-values.yaml @@ -29,6 +29,9 @@ externalSecrets: - secretKey: google-client-secret remoteRef: key: openbot/google-client-secret + - secretKey: computer-token + remoteRef: + key: openbot/computer-token - secretKey: license-token remoteRef: key: openbot-license-token @@ -42,3 +45,6 @@ httpRoute: namespace: gateway-system hostnames: - openbot.example.com + +computers: + mode: shared diff --git a/charts/openbot/ci/self-hosted-values.yaml b/charts/openbot/ci/self-hosted-values.yaml index 84720ba3..54701c72 100644 --- a/charts/openbot/ci/self-hosted-values.yaml +++ b/charts/openbot/ci/self-hosted-values.yaml @@ -21,6 +21,7 @@ secrets: keyEncryptionKey: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" intelligenceApiKey: "example-for-rendering-only" googleClientSecret: "example-for-rendering-only" + computerToken: "example-for-rendering-only" licenseToken: "example-for-rendering-only" ingress: enabled: true @@ -30,3 +31,6 @@ ingress: paths: - path: / pathType: Prefix + +computers: + mode: shared diff --git a/charts/openbot/templates/_helpers.tpl b/charts/openbot/templates/_helpers.tpl index 7f7a65a8..fe458c5e 100644 --- a/charts/openbot/templates/_helpers.tpl +++ b/charts/openbot/templates/_helpers.tpl @@ -152,9 +152,25 @@ and in whatever holds the release, which is not where `KEY_ENCRYPTION_KEY` belon - name: LOG_LEVEL value: {{ .Values.config.logLevel | quote }} {{- end }} -{{- if .Values.computers.url }} +{{- /* + Where this deployment's Bots find a computer, decided by the mode rather than by the operator. + + `shared` addresses the StatefulSet's one pod by its stable name, which is what a headless Service + gives it. `external` takes the URL as written. `sandbox` sets neither: the provider asks the + cluster for each Bot's own computer and gets an address back, so a fixed URL would be the one + thing that could send every Bot to the same browser. +*/}} +{{- if eq .Values.computers.mode "shared" }} +- name: AGENT_COMPUTER_URL + value: http://{{ include "openbot.componentName" (dict "root" . "component" "computer") }}-0.{{ include "openbot.componentName" (dict "root" . "component" "computer") }}:4100 +{{- else if and (eq .Values.computers.mode "external") .Values.computers.url }} - name: AGENT_COMPUTER_URL value: {{ .Values.computers.url | quote }} +{{- else if eq .Values.computers.mode "sandbox" }} +- name: COMPUTER_SANDBOX_NAMESPACE + value: {{ default .Release.Namespace .Values.computers.sandbox.namespace | quote }} +- name: COMPUTER_SANDBOX_IDLE_AFTER + value: {{ .Values.computers.sandbox.idleAfter | quote }} {{- end }} - name: INTELLIGENCE_API_URL value: {{ .Values.config.intelligence.apiUrl | quote }} @@ -227,7 +243,7 @@ and in whatever holds the release, which is not where `KEY_ENCRYPTION_KEY` belon secretKeyRef: name: {{ default (include "openbot.secretName" .) .Values.computers.existingTokenSecret }} key: computer-token - optional: true + optional: {{ eq .Values.computers.mode "external" }} {{- with .Values.config.extraEnv }} {{ toYaml . }} {{- end }} diff --git a/charts/openbot/templates/computer/culler-cronjob.yaml b/charts/openbot/templates/computer/culler-cronjob.yaml new file mode 100644 index 00000000..62e30d3f --- /dev/null +++ b/charts/openbot/templates/computer/culler-cronjob.yaml @@ -0,0 +1,59 @@ +{{- if and (eq .Values.computers.mode "sandbox") .Values.computers.sandbox.culler.enabled }} +{{- $component := "culler" -}} +{{/* +Suspending computers nobody is using. + +A CronJob rather than a timer in the API, and the difference is the whole reason this file exists. +An interval in the server fires in every replica, so five replicas would each decide independently +to suspend the same computer. The audit-retention sweep gets away with that because deleting old +rows twice is the same as deleting them once; suspending a browser somebody just started using is +not. The work is claimed and leased out of PostgreSQL, so whichever pod runs this takes what nobody +else holds, and a pod that dies mid-suspend hands its work back when the lease expires. + +`concurrencyPolicy: Forbid` on top, because a schedule that overlaps itself is the same problem in +one workload rather than across several. +*/}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} +spec: + schedule: {{ .Values.computers.sandbox.culler.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + startingDeadlineSeconds: 120 + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + labels: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 12 }} + spec: + restartPolicy: Never + serviceAccountName: {{ include "openbot.serviceAccountName" . }} + {{- /* It reads the cluster's Sandboxes, so it needs the token the API pod does not. */}} + automountServiceAccountToken: true + {{- with .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml . | indent 12 }} + {{- end }} + containers: + - name: culler + image: {{ include "openbot.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + workingDir: /app/server + command: ["/usr/local/bin/bun", "scripts/cull-idle-computers.ts"] + env: +{{ include "openbot.databaseUrlEnv" . | indent 16 }} +{{ include "openbot.commonEnv" . | indent 16 }} + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi +{{- end }} diff --git a/charts/openbot/templates/computer/sandbox-rbac.yaml b/charts/openbot/templates/computer/sandbox-rbac.yaml new file mode 100644 index 00000000..14e57a82 --- /dev/null +++ b/charts/openbot/templates/computer/sandbox-rbac.yaml @@ -0,0 +1,46 @@ +{{- if eq .Values.computers.mode "sandbox" }} +{{- $ns := default .Release.Namespace .Values.computers.sandbox.namespace -}} +{{/* +What the API may do to computers, and nothing else. + +A ROLE, NOT A CLUSTERROLE. Scoped to the one namespace the computers live in, so the worst this +credential can do is manage Bots' browsers in the place they already are. Compare the Docker +supervisor, which holds the host's Docker socket and is therefore root-equivalent on that host: this +is a smaller blast radius, granted by the cluster rather than by a shared environment variable. +*/}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" "sandbox") }} + namespace: {{ $ns }} + labels: +{{ include "openbot.labels" . | indent 4 }} +rules: + - apiGroups: ["agents.x-k8s.io"] + resources: ["sandboxes"] + verbs: ["get", "list", "watch", "create", "patch", "delete"] + - apiGroups: ["agents.x-k8s.io"] + resources: ["sandboxes/status"] + verbs: ["get"] + {{- if .Values.computers.sandbox.warmPool.enabled }} + - apiGroups: ["agents.x-k8s.io"] + resources: ["sandboxclaims"] + verbs: ["get", "list", "create", "delete"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" "sandbox") }} + namespace: {{ $ns }} + labels: +{{ include "openbot.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "openbot.componentName" (dict "root" . "component" "sandbox") }} +subjects: + - kind: ServiceAccount + name: {{ include "openbot.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/openbot/templates/computer/sandbox-template.yaml b/charts/openbot/templates/computer/sandbox-template.yaml new file mode 100644 index 00000000..d23a2418 --- /dev/null +++ b/charts/openbot/templates/computer/sandbox-template.yaml @@ -0,0 +1,105 @@ +{{- if eq .Values.computers.mode "sandbox" }} +{{- $ns := default .Release.Namespace .Values.computers.sandbox.namespace -}} +{{/* +The pod every Bot's computer is cut from. + +Here rather than in the server, because what image a computer runs, what volumes it keeps and what +runtime class it uses are deployment decisions, and the three clouds disagree about the last one. +The server only names this template; the cluster fills in the rest. +*/}} +apiVersion: agents.x-k8s.io/v1beta1 +kind: SandboxTemplate +metadata: + name: {{ $ns }}-computer + namespace: {{ $ns }} + labels: +{{ include "openbot.labels" . | indent 4 }} +spec: + podTemplate: + metadata: + labels: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" "computer") | indent 8 }} + spec: + {{- with .Values.computers.runtimeClassName }} + runtimeClassName: {{ . }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml . | indent 8 }} + {{- end }} + terminationGracePeriodSeconds: 30 + {{- with .Values.computers.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.computers.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: computer + image: {{ include "openbot.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/usr/local/bin/bun", "/app/agent-computer/src/index.ts"] + ports: + - name: http + containerPort: 4100 + env: + - name: PORT + value: "4100" + - name: WORKSPACE_DIR + value: /workspace + - name: PROFILES_DIR + value: /profiles + - name: COMPUTER_TOKEN + valueFrom: + secretKeyRef: + name: {{ default (include "openbot.secretName" .) .Values.computers.existingTokenSecret }} + key: computer-token + {{- with .Values.computers.extraEnv }} +{{ toYaml . | indent 12 }} + {{- end }} + volumeMounts: + - name: profiles + mountPath: /profiles + - name: workspace + mountPath: /workspace + {{- /* Readiness only. A browser slow under load is not a browser to restart. */}} + readinessProbe: + httpGet: + path: /health + port: http + periodSeconds: 10 + failureThreshold: 6 + {{- with .Values.computers.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} + {{- /* + The volumes that make a suspend worth doing. + + Suspending terminates the pod and keeps these, which is the difference between a computer that + comes back with its logins and one that comes back signed out of everything. + */}} + volumeClaimTemplates: + - metadata: + name: profiles + spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.computers.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.computers.persistence.profilesSize }} + - metadata: + name: workspace + spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.computers.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.computers.persistence.workspaceSize }} +{{- end }} diff --git a/charts/openbot/templates/computer/service.yaml b/charts/openbot/templates/computer/service.yaml new file mode 100644 index 00000000..8d4719da --- /dev/null +++ b/charts/openbot/templates/computer/service.yaml @@ -0,0 +1,19 @@ +{{- if eq .Values.computers.mode "shared" }} +{{- $component := "computer" -}} +{{/* Headless, because a StatefulSet wants stable per-pod names rather than a load-balanced one. */}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} +spec: + clusterIP: None + ports: + - port: 4100 + targetPort: http + protocol: TCP + name: http + selector: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 4 }} +{{- end }} diff --git a/charts/openbot/templates/computer/statefulset.yaml b/charts/openbot/templates/computer/statefulset.yaml new file mode 100644 index 00000000..a7f570aa --- /dev/null +++ b/charts/openbot/templates/computer/statefulset.yaml @@ -0,0 +1,161 @@ +{{- if eq .Values.computers.mode "shared" }} +{{- $component := "computer" -}} +{{/* +A Bot's computer: Chromium, a workspace, and a browser profile that has to survive a restart. + +A STATEFULSET, NOT A DEPLOYMENT, and not because there is more than one of them. The profile +directory holds real logins, and a Deployment's pods get no stable volume of their own: two replicas +would fight over one `ReadWriteOnce` claim and a rollout would hand the new pod an empty profile, +which reads as every Bot being signed out of everything at once. + +`replicas: 1` on purpose in this mode. One browser for every Bot is what `shared` means, and it is +the mode that needs no CRD in the cluster. `computers.mode: sandbox` gives each Bot its own, and is +where the idle suspend lives; see the provider RBAC beside this file. + +STORAGE HAS GRAVITY. The volume below is `ReadWriteOnce`, and on all three clouds the ordinary block +volume is zonal, so this pod is pinned to whichever zone its volume was created in for as long as +that profile exists. That is acceptable and worth stating rather than discovering. `storageClass` +stays empty by default, meaning the cluster's default class, because naming `gp3` or `pd-balanced` +here is how a chart stops installing on somebody's bare-metal cluster. +*/}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" $component) | indent 4 }} +spec: + serviceName: {{ include "openbot.componentName" (dict "root" . "component" $component) }} + replicas: 1 + selector: + matchLabels: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 6 }} + {{- /* + THE DEFAULT IS RETAIN, AND THAT IS WHAT WE WANT, SAID OUT LOUD. + + `whenScaled: Delete` here would erase every Bot's logins the first time this scaled down. It is + written explicitly rather than inherited, so that nobody later "tidies up" a field they think is + missing. + */}} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.computers.persistence.whenDeleted }} + whenScaled: Retain + volumeClaimTemplates: + - metadata: + name: profiles + spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.computers.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.computers.persistence.profilesSize }} + - metadata: + name: workspace + spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.computers.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.computers.persistence.workspaceSize }} + template: + metadata: + labels: +{{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml . | indent 8 }} + {{- end }} + {{- /* + gVisor or Kata, where the cluster has one, and an ordinary pod where it does not. + + The product already has this idea: `COMPUTER_RUNTIME=runsc` runs a computer under gVisor on a + host that supports it. This is the same setting where Kubernetes expects it. Unset by + default, because a chart that assumed a RuntimeClass would fail to install on a cluster that + has none, and the three clouds do not agree: GKE has managed gVisor, AKS offers Kata, and on + EKS it is bring your own node configuration. + */}} + {{- with .Values.computers.runtimeClassName }} + runtimeClassName: {{ . }} + {{- end }} + terminationGracePeriodSeconds: 30 + {{- with .Values.computers.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.computers.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: computer + image: {{ include "openbot.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- /* + The computer alone, not the whole image. + + One image serves both roles and the command decides which: this skips s6 and runs the + browser process directly, so a computer pod carries no API and no database. + */}} + command: ["/usr/local/bin/bun", "/app/agent-computer/src/index.ts"] + ports: + - name: http + containerPort: 4100 + protocol: TCP + env: + - name: PORT + value: "4100" + - name: WORKSPACE_DIR + value: /workspace + - name: PROFILES_DIR + value: /profiles + - name: COMPUTER_TOKEN + valueFrom: + secretKeyRef: + name: {{ default (include "openbot.secretName" .) .Values.computers.existingTokenSecret }} + key: computer-token + {{- with .Values.computers.maxBrowsers }} + - name: COMPUTER_MAX_BROWSERS + value: {{ . | quote }} + {{- end }} + {{- with .Values.computers.browserIdleMs }} + - name: COMPUTER_BROWSER_IDLE_MS + value: {{ . | quote }} + {{- end }} + {{- with .Values.computers.extraEnv }} +{{ toYaml . | indent 12 }} + {{- end }} + volumeMounts: + - name: profiles + mountPath: /profiles + - name: workspace + mountPath: /workspace + {{- /* + Readiness only, and deliberately no liveness probe. + + A browser under load can be slow to answer without being broken, and a liveness probe + that restarts it takes every signed-in session with it. Readiness keeps traffic away + until it can answer; nothing kills it for being busy. + */}} + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 6 + startupProbe: + httpGet: + path: /health + port: http + periodSeconds: 5 + failureThreshold: 60 + {{- with .Values.computers.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} +{{- end }} diff --git a/charts/openbot/templates/computer/warmpool.yaml b/charts/openbot/templates/computer/warmpool.yaml new file mode 100644 index 00000000..f91ab0b1 --- /dev/null +++ b/charts/openbot/templates/computer/warmpool.yaml @@ -0,0 +1,22 @@ +{{- if and (eq .Values.computers.mode "sandbox") .Values.computers.sandbox.warmPool.enabled }} +{{- $ns := default .Release.Namespace .Values.computers.sandbox.namespace -}} +{{/* +Computers waiting, so a Bot's first action after lunch does not wait for Chromium to boot. + +A resume costs a pod schedule and a browser launch, and a `ReadWriteOnce` volume that has to detach +from the old node first. That is a real wait, and this is the upstream mechanism for avoiding most +of it rather than something to reinvent. Off by default, because a pool is browsers nobody is using +yet, which is exactly the cost the suspend is here to remove. +*/}} +apiVersion: agents.x-k8s.io/v1beta1 +kind: SandboxWarmPool +metadata: + name: {{ $ns }}-computer + namespace: {{ $ns }} + labels: +{{ include "openbot.labels" . | indent 4 }} +spec: + replicas: {{ .Values.computers.sandbox.warmPool.replicas }} + sandboxTemplateRef: + name: {{ $ns }}-computer +{{- end }} diff --git a/charts/openbot/templates/validation.yaml b/charts/openbot/templates/validation.yaml index c25847a2..c7ec4146 100644 --- a/charts/openbot/templates/validation.yaml +++ b/charts/openbot/templates/validation.yaml @@ -67,6 +67,26 @@ This template renders nothing. {{- fail "OpenBot requires CopilotKit Intelligence. Set config.intelligence.apiUrl and config.intelligence.gatewayWsUrl, and the matching secrets.intelligenceApiKey and secrets.licenseToken." }} {{- end }} +{{- /* + A computer nobody can reach, or one anybody can. + + `agent-computer` refuses every request without `COMPUTER_TOKEN` and permits only `/health`, so a + chart that created one without a token would create a process that answers nothing. And a mode of + `external` with no address is a deployment whose Bots have no computer at all, which fails at the + first browser action rather than at install. +*/}} +{{- if and (ne .Values.computers.mode "external") (not .Values.secrets.computerToken) (not .Values.computers.existingTokenSecret) (not .Values.secrets.existingSecret) (not .Values.externalSecrets.enabled) }} +{{- fail "A computer needs a token, which is the only thing standing between it and anybody who can reach its port. Set secrets.computerToken. Generate one with: openssl rand -hex 32" }} +{{- end }} + +{{- if and (eq .Values.computers.mode "external") (not .Values.computers.url) }} +{{- fail "computers.mode is external but computers.url is empty, so no Bot would have a computer. Set the address, or use mode: shared to have this chart run one." }} +{{- end }} + +{{- if not (has .Values.computers.mode (list "shared" "sandbox" "external")) }} +{{- fail "computers.mode must be one of: shared, sandbox, external." }} +{{- end }} + {{- if and .Values.ingress.enabled .Values.httpRoute.enabled }} {{- fail "ingress.enabled and httpRoute.enabled are both set. They are two ways to do the same thing; pick the one your cluster runs." }} {{- end }} diff --git a/charts/openbot/values.yaml b/charts/openbot/values.yaml index dc9107d5..e4137214 100644 --- a/charts/openbot/values.yaml +++ b/charts/openbot/values.yaml @@ -148,14 +148,67 @@ config: extraEnv: [] extraEnvFrom: [] -# How a Bot gets a computer. `shared` is one browser for every Bot, which is the only shape this -# slice of the chart ships; per-Bot computers arrive with the Sandbox provider. +# How a Bot gets a computer. +# +# shared One browser for every Bot, run by this chart. Needs nothing installed in the cluster, +# and is what a first install should use. +# sandbox A computer each, as a `Sandbox` from kubernetes-sigs/agent-sandbox, suspended when +# idle and resumed with its logins intact. Needs that controller in the cluster. +# external Neither. `url` points at a computer somebody else runs. computers: mode: shared + # Only for `mode: external`. The other modes derive the address from what they created. url: "" # Never a literal. See `secrets` below. existingTokenSecret: "" + # gVisor or Kata, where the cluster has one. Unset means an ordinary pod, because the three clouds + # do not agree: GKE has managed gVisor, AKS offers Kata and gVisor, and on EKS it is bring your own + # node configuration. A chart that assumed a RuntimeClass would fail to install without one. + runtimeClassName: "" + + # How many browsers one computer holds at once, and how long an untouched one is kept. Empty means + # the process default, which is a handful and thirty minutes. + maxBrowsers: "" + browserIdleMs: "" + + persistence: + # EMPTY MEANS THE CLUSTER'S DEFAULT CLASS. Naming `gp3` or `pd-balanced` is how a chart stops + # installing on a bare-metal cluster. Note that the ordinary block volume on every cloud is + # zonal, so a computer is pinned to the zone its profile was created in. + storageClass: "" + profilesSize: 10Gi + workspaceSize: 10Gi + # What happens to a Bot's logins when the release is deleted. `Retain` keeps them, which is the + # safe default; `Delete` is for a trial you want to leave no trace of. + whenDeleted: Retain + + resources: + requests: + cpu: 500m + # Chromium is roughly 200-500MB headless and up to 2GB with real pages open. + memory: 1Gi + limits: + memory: 4Gi + + nodeSelector: {} + tolerations: [] + extraEnv: [] + + # `mode: sandbox` only. Where the per-Bot computers are created and what may create them. + sandbox: + namespace: "" + # Pre-warmed sandboxes, so a Bot's first action after lunch does not wait for Chromium to boot. + warmPool: + enabled: false + replicas: 2 + # Suspend a computer nobody has used for this long. The Sandbox CRD has an absolute expiry, which + # is not the same question, so the culler below asks this one. + idleAfter: 30m + culler: + enabled: true + schedule: "*/5 * * * *" + database: # Used when `postgresql.enabled` is false. A URL, or a secret holding one. url: "" diff --git a/server/drizzle.config.ts b/server/drizzle.config.ts index 24e6775d..a0fdf026 100644 --- a/server/drizzle.config.ts +++ b/server/drizzle.config.ts @@ -23,6 +23,7 @@ export default defineConfig({ "./src/db/schema/coworker.ts", "./src/db/schema/components.ts", "./src/db/schema/plugins.ts", + "./src/db/schema/work.ts", ], out: "./drizzle", dbCredentials: { diff --git a/server/drizzle/0016_durable_work.sql b/server/drizzle/0016_durable_work.sql new file mode 100644 index 00000000..5cb2e9af --- /dev/null +++ b/server/drizzle/0016_durable_work.sql @@ -0,0 +1,14 @@ +CREATE TABLE "work_items" ( + "kind" text NOT NULL, + "key" text NOT NULL, + "run_at" timestamp with time zone DEFAULT now() NOT NULL, + "claimed_by" text, + "lease_until" timestamp with time zone, + "attempts" integer DEFAULT 0 NOT NULL, + "payload" jsonb DEFAULT '{}'::jsonb NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "work_items_kind_key_pk" PRIMARY KEY("kind","key") +); +--> statement-breakpoint +CREATE INDEX "work_items_claimable_idx" ON "work_items" USING btree ("kind","run_at"); \ No newline at end of file diff --git a/server/drizzle/meta/0016_snapshot.json b/server/drizzle/meta/0016_snapshot.json new file mode 100644 index 00000000..151e0740 --- /dev/null +++ b/server/drizzle/meta/0016_snapshot.json @@ -0,0 +1,2482 @@ +{ + "id": "0333e21d-0b63-4aac-9234-6ba46382f2b2", + "prevId": "8f1f91d5-9cb1-490a-a28f-a0a3f15b9614", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_provider_account_idx": { + "name": "accounts_provider_account_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "agent_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_package_id_deployment_packages_id_fk": { + "name": "agents_package_id_deployment_packages_id_fk", + "tableFrom": "agents", + "tableTo": "deployment_packages", + "columnsFrom": ["package_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_events": { + "name": "audit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_events_created_at_idx": { + "name": "audit_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_type_time_idx": { + "name": "audit_events_type_time_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_actor_time_idx": { + "name": "audit_events_actor_time_idx", + "columns": [ + { + "expression": "actor_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_target_time_idx": { + "name": "audit_events_target_time_idx", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_agents": { + "name": "channel_agents", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_agents_channel_id_channels_id_fk": { + "name": "channel_agents_channel_id_channels_id_fk", + "tableFrom": "channel_agents", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_agents_agent_id_agents_id_fk": { + "name": "channel_agents_agent_id_agents_id_fk", + "tableFrom": "channel_agents", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_agents_channel_id_agent_id_pk": { + "name": "channel_agents_channel_id_agent_id_pk", + "columns": ["channel_id", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_memberships": { + "name": "channel_memberships", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_memberships_channel_id_channels_id_fk": { + "name": "channel_memberships_channel_id_channels_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_memberships_user_id_users_id_fk": { + "name": "channel_memberships_user_id_users_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_memberships_channel_id_user_id_pk": { + "name": "channel_memberships_channel_id_user_id_pk", + "columns": ["channel_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channels": { + "name": "channels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suggested_prompts": { + "name": "suggested_prompts", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "allowed_groups": { + "name": "allowed_groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_message": { + "name": "last_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_message_at": { + "name": "last_message_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message_agent_id": { + "name": "last_message_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "channels_recent_activity_idx": { + "name": "channels_recent_activity_idx", + "columns": [ + { + "expression": "COALESCE(\"last_message_at\", \"created_at\") DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "channels_package_id_deployment_packages_id_fk": { + "name": "channels_package_id_deployment_packages_id_fk", + "tableFrom": "channels", + "tableTo": "deployment_packages", + "columnsFrom": ["package_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "channels_last_message_agent_id_agents_id_fk": { + "name": "channels_last_message_agent_id_agents_id_fk", + "tableFrom": "channels", + "tableTo": "agents", + "columnsFrom": ["last_message_agent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credentials": { + "name": "credentials", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "credential_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_value": { + "name": "encrypted_value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credentials_active_key_idx": { + "name": "credentials_active_key_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credentials\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_packages": { + "name": "deployment_packages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "loaded_at": { + "name": "loaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployment_packages_tenant_id_unique": { + "name": "deployment_packages_tenant_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.intelligence_channel_mappings": { + "name": "intelligence_channel_mappings", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "intelligence_channel_mappings_thread_idx": { + "name": "intelligence_channel_mappings_thread_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "intelligence_channel_mappings_user_id_users_id_fk": { + "name": "intelligence_channel_mappings_user_id_users_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "intelligence_channel_mappings_channel_id_channels_id_fk": { + "name": "intelligence_channel_mappings_channel_id_channels_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "intelligence_channel_mappings_user_id_channel_id_pk": { + "name": "intelligence_channel_mappings_user_id_channel_id_pk", + "columns": ["user_id", "channel_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.revoked_access": { + "name": "revoked_access", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_providers": { + "name": "sso_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sso_providers_user_id_users_id_fk": { + "name": "sso_providers_user_id_users_id_fk", + "tableFrom": "sso_providers", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "nullsNotDistinct": false, + "columns": ["provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_pk": { + "name": "user_roles_user_id_role_pk", + "columns": ["user_id", "role"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "groups": { + "name": "groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verifications": { + "name": "verifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_policy": { + "name": "action_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deny": { + "name": "deny", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "allow": { + "name": "allow", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_snapshot": { + "name": "computer_snapshot", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "elements": { + "name": "elements", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "taken_at": { + "name": "taken_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "session": { + "name": "session", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_preferences": { + "name": "agent_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_preferences_user_id_users_id_fk": { + "name": "agent_preferences_user_id_users_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_preferences_agent_id_agents_id_fk": { + "name": "agent_preferences_agent_id_agents_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_preferences_user_id_agent_id_pk": { + "name": "agent_preferences_user_id_agent_id_pk", + "columns": ["user_id", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_profiles": { + "name": "agent_profiles", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_description": { + "name": "role_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_seed": { + "name": "avatar_seed", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "agent_visibility", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "callback_token_hash": { + "name": "callback_token_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "callback_token_issued_at": { + "name": "callback_token_issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_profiles_visibility_deleted_idx": { + "name": "agent_profiles_visibility_deleted_idx", + "columns": [ + { + "expression": "visibility", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_profiles_agent_id_agents_id_fk": { + "name": "agent_profiles_agent_id_agents_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_profiles_owner_user_id_users_id_fk": { + "name": "agent_profiles_owner_user_id_users_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_exclusions": { + "name": "component_exclusions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "withheld_by": { + "name": "withheld_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_exclusions_component_name_components_name_fk": { + "name": "component_exclusions_component_name_components_name_fk", + "tableFrom": "component_exclusions", + "tableTo": "components", + "columnsFrom": ["component_name"], + "columnsTo": ["name"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "component_exclusions_agent_id_agents_id_fk": { + "name": "component_exclusions_agent_id_agents_id_fk", + "tableFrom": "component_exclusions", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_exclusions_component_name_agent_id_pk": { + "name": "component_exclusions_component_name_agent_id_pk", + "columns": ["component_name", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_functions": { + "name": "component_functions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "function_name": { + "name": "function_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_functions_component_name_components_name_fk": { + "name": "component_functions_component_name_components_name_fk", + "tableFrom": "component_functions", + "tableTo": "components", + "columnsFrom": ["component_name"], + "columnsTo": ["name"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_functions_component_name_function_name_pk": { + "name": "component_functions_component_name_function_name_pk", + "columns": ["component_name", "function_name"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.components": { + "name": "components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'first-party'" + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tools_refreshed_at": { + "name": "tools_refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_servers_credential_id_credentials_id_fk": { + "name": "mcp_servers_credential_id_credentials_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "credentials", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_tools": { + "name": "mcp_tools", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "input_schema": { + "name": "input_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_tools_server_id_mcp_servers_id_fk": { + "name": "mcp_tools_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_tools", + "tableTo": "mcp_servers", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_tools_server_id_name_pk": { + "name": "mcp_tools_server_id_name_pk", + "columns": ["server_id", "name"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_user_credentials": { + "name": "mcp_user_credentials", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_user_credentials_user_idx": { + "name": "mcp_user_credentials_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_user_credentials_server_id_mcp_servers_id_fk": { + "name": "mcp_user_credentials_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "mcp_servers", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_user_id_users_id_fk": { + "name": "mcp_user_credentials_user_id_users_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_credential_id_credentials_id_fk": { + "name": "mcp_user_credentials_credential_id_credentials_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "credentials", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_user_credentials_server_id_user_id_pk": { + "name": "mcp_user_credentials_server_id_user_id_pk", + "columns": ["server_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_grants": { + "name": "plugin_grants", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "plugin_grants_agent_idx": { + "name": "plugin_grants_agent_idx", + "columns": [ + { + "expression": "agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "plugin_grants_agent_id_agents_id_fk": { + "name": "plugin_grants_agent_id_agents_id_fk", + "tableFrom": "plugin_grants", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "plugin_grants_kind_ref_agent_id_pk": { + "name": "plugin_grants_kind_ref_agent_id_pk", + "columns": ["kind", "ref", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandboxed_components": { + "name": "sandboxed_components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_html": { + "name": "draft_html", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_css": { + "name": "draft_css", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_js_functions": { + "name": "draft_js_functions", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_argument_schema": { + "name": "draft_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_html": { + "name": "published_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_css": { + "name": "published_css", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_js_functions": { + "name": "published_js_functions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_argument_schema": { + "name": "published_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "sample_arguments": { + "name": "sample_arguments", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "authored_by": { + "name": "authored_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_tools": { + "name": "skill_tools", + "schema": "", + "columns": { + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "declared_by": { + "name": "declared_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_tools_ref_idx": { + "name": "skill_tools_ref_idx", + "columns": [ + { + "expression": "ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_tools_skill_id_skills_id_fk": { + "name": "skill_tools_skill_id_skills_id_fk", + "tableFrom": "skill_tools", + "tableTo": "skills", + "columnsFrom": ["skill_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "skill_tools_skill_id_ref_pk": { + "name": "skill_tools_skill_id_ref_pk", + "columns": ["skill_id", "ref"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skills": { + "name": "skills", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'yours'" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skills_slug_key": { + "name": "skills_slug_key", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skills_owner_idx": { + "name": "skills_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skills_owner_user_id_users_id_fk": { + "name": "skills_owner_user_id_users_id_fk", + "tableFrom": "skills", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.work_items": { + "name": "work_items", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_at": { + "name": "run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "work_items_claimable_idx": { + "name": "work_items_claimable_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "work_items_kind_key_pk": { + "name": "work_items_kind_key_pk", + "columns": ["kind", "key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.agent_type": { + "name": "agent_type", + "schema": "public", + "values": ["built_in", "remote_ag_ui"] + }, + "public.credential_kind": { + "name": "credential_kind", + "schema": "public", + "values": [ + "model", + "connector", + "agent", + "mcp", + "mcp_oauth_client", + "mcp_user_token" + ] + }, + "public.role": { + "name": "role", + "schema": "public", + "values": ["admin", "user"] + }, + "public.agent_visibility": { + "name": "agent_visibility", + "schema": "public", + "values": ["public", "private"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/server/drizzle/meta/_journal.json b/server/drizzle/meta/_journal.json index 1162cae9..ebd2290e 100644 --- a/server/drizzle/meta/_journal.json +++ b/server/drizzle/meta/_journal.json @@ -113,6 +113,13 @@ "when": 1787525879804, "tag": "0015_credentials_one_live_key", "breakpoints": true + }, + { + "idx": 16, + "version": "7", + "when": 1787602199792, + "tag": "0016_durable_work", + "breakpoints": true } ] } diff --git a/server/scripts/cull-idle-computers.ts b/server/scripts/cull-idle-computers.ts new file mode 100644 index 00000000..5bb888e6 --- /dev/null +++ b/server/scripts/cull-idle-computers.ts @@ -0,0 +1,62 @@ +/** + * One sweep: notice which computers have gone idle, and suspend whatever this pod can claim. + * + * Run from a CronJob rather than from a timer inside the API. Every replica would fire its own timer + * and each would decide, independently, to suspend the same computer. Deleting old audit rows twice + * is harmless, which is why the retention sweep may work that way; taking a browser away from + * somebody who has just come back is not. + * + * Exits non-zero only when the sweep itself could not run. A computer that refused to suspend is + * reported and left for the next sweep, because a computer still running costs money rather than + * losing anything, and a failing CronJob that pages somebody at 3am should mean something worse. + */ +import { randomUUID } from "node:crypto"; +import { createComputerProvider } from "../src/computer/provider"; +import { loadConfig } from "../src/config"; +import { createDatabase } from "../src/db/client"; +import { + offerIdleComputers, + suspendClaimedComputers, +} from "../src/work/culler"; +import { createWorkQueue } from "../src/work/queue"; + +const config = loadConfig(process.env); +if (!config.computer) { + throw new Error( + "No computer provider is configured, so there are no computers to suspend.", + ); +} +if (config.computer.provider !== "sandbox") { + throw new Error( + `The culler only has something to do where each Bot has its own computer, and this deployment uses the "${config.computer.provider}" provider.`, + ); +} + +const database = createDatabase(config.databaseUrl); +const queue = createWorkQueue(database); +const provider = createComputerProvider(config.computer); + +// A name for the lease, so a stuck claim can be traced back to the pod that took it. +const owner = `culler/${process.env.HOSTNAME ?? randomUUID().slice(0, 8)}`; + +try { + const options = { + database, + queue, + provider, + idleAfterMs: config.computer.idleAfterMs, + owner, + }; + const { offered } = await offerIdleComputers(options); + const report = await suspendClaimedComputers(options); + console.info( + JSON.stringify({ + type: "computer-cull", + offered, + suspended: report.suspended, + skipped: report.skipped, + }), + ); +} finally { + await database.$client.end({ timeout: 5 }); +} diff --git a/server/src/computer/provider.ts b/server/src/computer/provider.ts index 7993227c..4a24b8fe 100644 --- a/server/src/computer/provider.ts +++ b/server/src/computer/provider.ts @@ -1,11 +1,11 @@ import type { ComputerConfig } from "../config"; +import { createSandboxComputerProvider, inClusterConfig } from "./sandbox"; +import type { ComputerStatus } from "./schema"; import { createDockerSupervisorProvider, type SupervisorOptions, } from "./supervisor"; -import type { ComputerStatus } from "./schema"; - /** The address and lifecycle details for one Bot's computer. */ export type ComputerLocation = { botId: string; @@ -232,5 +232,65 @@ export function createComputerProvider( baseUrl: config.baseUrl, ...(config.token ? { token: config.token } : {}), }); + case "sandbox": + /* + * Built lazily, because reading the service account is asynchronous and this factory is not. + * Every method needs the same credentials, so the promise is created once and awaited by each + * rather than the file being read on every call. + */ + return createLazySandboxProvider(config); } } + +/** + * The sandbox provider, built on first use. + * + * Its credentials come off disk, which is asynchronous, and `createComputerProvider` is not. Rather + * than make every caller await a factory, the work happens once behind a promise and each method + * waits on the same one. A failure to read them surfaces on the first computer request, naming what + * is missing, instead of taking the whole deployment down at boot over a feature it may not use. + */ +function createLazySandboxProvider( + config: Extract, +): ComputerProvider { + let built: Promise | undefined; + + const provider = async (): Promise => { + built ??= (async () => { + const cluster = await inClusterConfig(); + return createSandboxComputerProvider({ + namespace: config.namespace, + idleAfterMs: config.idleAfterMs, + template: sandboxPodTemplate(config), + apiServer: cluster.apiServer, + token: cluster.token, + ca: cluster.ca, + }); + })(); + return built; + }; + + return { + name: "sandbox", + isolation: "per-bot", + locate: async (botId) => (await provider()).locate(botId), + status: async (botId) => (await provider()).status(botId), + stop: async (botId) => (await provider()).stop(botId), + reset: async (botId) => (await provider()).reset(botId), + list: async () => (await provider()).list(), + sessionOf: async (botId) => (await provider()).sessionOf?.(botId), + }; +} + +/** + * The pod every Bot's computer is cut from. + * + * Read from the `SandboxTemplate` the chart installs rather than written here: what image a computer + * runs, what volumes it keeps and what runtime class it uses are deployment decisions, and the three + * clouds disagree about the last one. The server only needs to know that a template exists. + */ +function sandboxPodTemplate( + config: Extract, +): Record { + return { sandboxTemplateRef: { name: `${config.namespace}-computer` } }; +} diff --git a/server/src/computer/sandbox.ts b/server/src/computer/sandbox.ts new file mode 100644 index 00000000..4f851759 --- /dev/null +++ b/server/src/computer/sandbox.ts @@ -0,0 +1,318 @@ +/** + * A computer each, as a `Sandbox` on Kubernetes. + * + * `kubernetes-sigs/agent-sandbox` defines a CRD for "isolated, stateful singleton workloads with + * stable identity and persistent storage", aimed at agents that run untrusted code and drive + * graphical interfaces. That is a description of a Bot's computer, so this provider maps onto it + * rather than hand-rolling a StatefulSet per Bot and owning suspend, resume and identity ourselves. + * + * The part that would have been the hard build is a field. `spec.operatingMode` is `Running` or + * `Suspended`, and Suspended terminates the pod while keeping the volumes, so "spin down when idle, + * come back with the logins intact" is one patch and a condition to wait on. + * + * NO CLIENT LIBRARY. The Kubernetes API is HTTP and JSON, this needs five verbs of it, and a + * generated client would be a large dependency in an image that already ships a browser. The + * in-cluster service account gives a token and a CA, which is what `inClusterConfig` reads. + */ +import { readFile } from "node:fs/promises"; +import type { ComputerLocation, ComputerProvider } from "./provider"; +import type { ComputerStatus } from "./schema"; + +const SERVICE_ACCOUNT = "/var/run/secrets/kubernetes.io/serviceaccount"; +const GROUP = "agents.x-k8s.io"; +const VERSION = "v1beta1"; + +export class SandboxError extends Error { + constructor(message: string) { + super(message); + this.name = "SandboxError"; + } +} + +/** One Sandbox, in the shape the parts of it this file reads. */ +type Sandbox = { + metadata?: { name?: string; creationTimestamp?: string }; + spec?: { operatingMode?: "Running" | "Suspended" }; + status?: { + serviceFQDN?: string; + conditions?: { type?: string; status?: string; reason?: string }[]; + podIPs?: string[]; + nodeName?: string; + }; +}; + +export type SandboxProviderOptions = { + /** Where the Bots' computers live. The provider is scoped to exactly this namespace. */ + namespace: string; + /** The pod template every computer is cut from, as YAML-derived JSON from the chart. */ + template: Record; + /** How long a computer may go untouched before the culler suspends it. */ + idleAfterMs: number; + apiServer?: string; + token?: string; + ca?: string; + fetchImpl?: typeof fetch; + /** How long `locate` waits for a suspended computer to come back before giving up. */ + resumeTimeoutMs?: number; +}; + +/** + * The service account this pod was given, which is how it reaches the API server. + * + * Absent outside a cluster, and that is not an error here: `createComputerProvider` only asks for + * this provider when a deployment configured it, and a clear message about a missing token beats a + * connection refused to an address nobody set. + */ +export async function inClusterConfig(): Promise<{ + apiServer: string; + token: string; + ca: string; +}> { + const host = process.env.KUBERNETES_SERVICE_HOST; + const port = process.env.KUBERNETES_SERVICE_PORT ?? "443"; + if (!host) { + throw new SandboxError( + "COMPUTER_PROVIDER=sandbox needs to run inside a cluster: KUBERNETES_SERVICE_HOST is not set, so there is no API server to ask for a Bot's computer.", + ); + } + const [token, ca] = await Promise.all([ + readFile(`${SERVICE_ACCOUNT}/token`, "utf8"), + readFile(`${SERVICE_ACCOUNT}/ca.crt`, "utf8"), + ]); + return { apiServer: `https://${host}:${port}`, token: token.trim(), ca }; +} + +/** + * A Kubernetes name for a Bot. + * + * Bot ids are ours and may hold anything a person typed; a resource name may hold lowercase + * alphanumerics and dashes, and is refused rather than truncated by the API server. Anything else + * becomes a dash, and a short hash keeps two ids that differ only in punctuation from colliding on + * one computer, which would be one Bot reading another's logins. + */ +export function sandboxNameFor(botId: string): string { + const slug = botId + .toLowerCase() + .replace(/[^a-z0-9-]/g, "-") + .replace(/^-+|-+$/g, ""); + let hash = 5381; + for (let index = 0; index < botId.length; index += 1) { + hash = ((hash << 5) + hash + botId.charCodeAt(index)) >>> 0; + } + const suffix = hash.toString(36); + return `bot-${slug.slice(0, 40) || "unnamed"}-${suffix}`; +} + +function conditionOf(sandbox: Sandbox, type: string): string | undefined { + return sandbox.status?.conditions?.find((c) => c.type === type)?.status; +} + +/** Ready means the pod is up and the service answers; anything else is not somewhere to send a Bot. */ +function isReady(sandbox: Sandbox): boolean { + return conditionOf(sandbox, "Ready") === "True"; +} + +function isSuspended(sandbox: Sandbox): boolean { + return ( + sandbox.spec?.operatingMode === "Suspended" || + conditionOf(sandbox, "Suspended") === "True" + ); +} + +export function createSandboxComputerProvider( + options: SandboxProviderOptions, +): ComputerProvider { + const doFetch = options.fetchImpl ?? fetch; + const resumeTimeoutMs = options.resumeTimeoutMs ?? 120_000; + const base = () => + `${options.apiServer}/apis/${GROUP}/${VERSION}/namespaces/${options.namespace}/sandboxes`; + + async function call( + path: string, + init: RequestInit & { contentType?: string } = {}, + ): Promise { + const { contentType, ...rest } = init; + const response = await doFetch(`${base()}${path}`, { + ...rest, + headers: { + ...(options.token ? { authorization: `Bearer ${options.token}` } : {}), + ...(contentType ? { "content-type": contentType } : {}), + accept: "application/json", + ...(rest.headers ?? {}), + }, + }); + if (response.status === 404) return undefined; + if (!response.ok) { + const body = await response.text().catch(() => ""); + throw new SandboxError( + `The cluster refused a sandbox request (${response.status}): ${body.slice(0, 300)}`, + ); + } + return response.json(); + } + + const read = (botId: string) => + call(`/${sandboxNameFor(botId)}`) as Promise; + + /** The desired body for a Bot's computer. Created once, then only ever patched. */ + function desired(botId: string): Record { + return { + apiVersion: `${GROUP}/${VERSION}`, + kind: "Sandbox", + metadata: { + name: sandboxNameFor(botId), + namespace: options.namespace, + labels: { + "app.kubernetes.io/managed-by": "openbot", + "openbot.dev/component": "computer", + }, + // The Bot id as written, which the name above cannot always carry. The culler reads this + // rather than trying to reverse the slug. + annotations: { "openbot.dev/bot-id": botId }, + }, + spec: { operatingMode: "Running", ...options.template }, + }; + } + + async function waitForReady(botId: string): Promise { + const deadline = Date.now() + resumeTimeoutMs; + for (;;) { + const sandbox = await read(botId); + if (sandbox && isReady(sandbox) && sandbox.status?.serviceFQDN) { + return sandbox; + } + if (Date.now() >= deadline) { + throw new SandboxError( + `The computer for ${botId} did not become ready within ${Math.round(resumeTimeoutMs / 1000)}s. A resume costs a pod schedule and a browser launch, so this is a real wait rather than a failure, but it has to end somewhere.`, + ); + } + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } + } + + return { + name: "sandbox", + isolation: "per-bot", + + async locate(botId: string): Promise { + const existing = await read(botId); + if (!existing) { + await call("", { + method: "POST", + contentType: "application/json", + body: JSON.stringify(desired(botId)), + }); + } else if (isSuspended(existing)) { + /* + * Woken, because somebody is asking for it. + * + * `locate` runs immediately before an action, so reaching a suspended computer here means a + * person is waiting. A merge patch rather than a replace: the controller owns most of this + * object and writing the whole thing back would fight it. + */ + await call(`/${sandboxNameFor(botId)}`, { + method: "PATCH", + contentType: "application/merge-patch+json", + body: JSON.stringify({ spec: { operatingMode: "Running" } }), + }); + } + + const ready = await waitForReady(botId); + const fqdn = ready.status?.serviceFQDN; + if (!fqdn) { + throw new SandboxError( + `The computer for ${botId} is ready but reported no address, so it cannot be reached.`, + ); + } + return `http://${fqdn}:4100`; + }, + + async status(botId: string): Promise { + try { + const sandbox = await read(botId); + if (!sandbox) return { botId, state: "absent" }; + /* + * A SUSPENDED COMPUTER IS DOWN AND FINE, and reading it any other way is how scale-to-zero + * is lost. Answering this by dialling the pod would wake it, so every computer anything ever + * asked about would come back up and the bill would never fall. The conditions are the whole + * answer and nothing here touches the browser. + */ + if (isSuspended(sandbox)) return { botId, state: "absent" }; + if (isReady(sandbox)) return { botId, state: "ready" }; + return { botId, state: "starting" }; + } catch (error) { + return { + botId, + state: "unreachable", + reason: + error instanceof Error && error.message.length > 0 + ? error.message + : "The cluster could not be asked about this computer.", + }; + } + }, + + async stop(botId: string): Promise<{ wasRunning: boolean }> { + const sandbox = await read(botId); + if (!sandbox || isSuspended(sandbox)) return { wasRunning: false }; + // Suspended, not deleted: the pod goes and the volumes stay, which is the difference between + // stopping a computer and wiping a Bot's logins. + await call(`/${sandboxNameFor(botId)}`, { + method: "PATCH", + contentType: "application/merge-patch+json", + body: JSON.stringify({ spec: { operatingMode: "Suspended" } }), + }); + return { wasRunning: true }; + }, + + async reset(botId: string): Promise<{ cleared: boolean }> { + const sandbox = await read(botId); + if (!sandbox) return { cleared: false }; + // Deleted, which takes the volumes with it. This is the one that is meant to lose the logins. + await call(`/${sandboxNameFor(botId)}`, { method: "DELETE" }); + return { cleared: true }; + }, + + async list(): Promise { + const body = (await call("")) as { items?: Sandbox[] } | undefined; + return (body?.items ?? []).map((sandbox) => { + const botId = + (sandbox.metadata as { annotations?: Record }) + ?.annotations?.["openbot.dev/bot-id"] ?? + sandbox.metadata?.name ?? + ""; + return { + botId, + status: + isSuspended(sandbox) || !isReady(sandbox) ? "stopped" : "running", + url: sandbox.status?.serviceFQDN + ? `http://${sandbox.status.serviceFQDN}:4100` + : "", + ...(sandbox.metadata?.creationTimestamp + ? { startedAt: sandbox.metadata.creationTimestamp } + : {}), + }; + }); + }, + + async sessionOf(botId: string): Promise { + /* + * Which run of this computer this is, and it has to change across a suspend and resume. + * + * A snapshot's generation only orders snapshots within one run of a browser: a resumed + * computer counts from one again, so a ref the model still holds from before the suspend would + * match a row nothing has overwritten and the boundary would decide about an element on a page + * that no longer exists. The pod's address changes when it is rescheduled, and the node it + * landed on with it, so the two together identify the run without asking the browser anything. + * + * Reading, never ensuring: this must not be the thing that wakes a computer up. + */ + const sandbox = await read(botId); + if (!sandbox || isSuspended(sandbox)) return undefined; + const ip = sandbox.status?.podIPs?.[0]; + const node = sandbox.status?.nodeName; + if (!ip && !node) return undefined; + return [node, ip].filter(Boolean).join("/"); + }, + }; +} diff --git a/server/src/config.ts b/server/src/config.ts index c4a71671..ad4422d4 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -38,7 +38,26 @@ export type SharedComputerConfig = { policy?: ActionPolicy; }; -export type ComputerConfig = DockerComputerConfig | SharedComputerConfig; +/** + * A computer each, created by the cluster. + * + * The namespace is the whole scope: the service account this runs under may manage Sandboxes there + * and nowhere else, which is a smaller blast radius than the Docker supervisor's, since that one + * holds a socket that is root-equivalent on its host. + */ +export type SandboxComputerConfig = { + provider: "sandbox"; + namespace: string; + idleAfterMs: number; + token?: string; + allowPrivateHosts: boolean; + policy?: ActionPolicy; +}; + +export type ComputerConfig = + | DockerComputerConfig + | SharedComputerConfig + | SandboxComputerConfig; /** * Who a deployment lets in, and through which front door. @@ -560,10 +579,39 @@ function privateHostsAllowed(environment: Environment): boolean { return true; } +/** + * A duration a person would write, as milliseconds. + * + * `30m` rather than `1800000`, because this one is read and edited by whoever is deciding how long a + * computer may sit idle, and a wrong number of zeroes there is either a computer that never sleeps + * or one that vanishes mid-task. Plain digits are still milliseconds, so anything already set keeps + * its meaning. + */ +export function durationMs(value: string): number { + const match = /^(\d+)\s*(ms|s|m|h)?$/.exec(value.trim()); + if (!match) { + throw new Error( + `"${value}" is not a duration. Write it as 30s, 30m, 2h, or a plain number of milliseconds.`, + ); + } + const amount = Number(match[1]); + switch (match[2]) { + case "h": + return amount * 3_600_000; + case "m": + return amount * 60_000; + case "s": + return amount * 1_000; + default: + return amount; + } +} + function computerConfig(environment: Environment): ComputerConfig | undefined { const supervisorAddress = optional(environment, "COMPUTER_SUPERVISOR_URL"); const sharedAddress = optional(environment, "AGENT_COMPUTER_URL"); - if (!supervisorAddress && !sharedAddress) { + const sandboxNamespace = optional(environment, "COMPUTER_SANDBOX_NAMESPACE"); + if (!supervisorAddress && !sharedAddress && !sandboxNamespace) { return undefined; } @@ -577,6 +625,24 @@ function computerConfig(environment: Environment): ComputerConfig | undefined { const allowPrivateHosts = privateHostsAllowed(environment); const policy = actionPolicy(environment); + /* + * Checked before the other two, because a deployment that named a namespace means the cluster to + * make the computers, and a stray `AGENT_COMPUTER_URL` left in an environment would otherwise + * quietly put every Bot back on one shared browser. + */ + if (sandboxNamespace) { + return { + provider: "sandbox", + namespace: sandboxNamespace, + idleAfterMs: durationMs( + optional(environment, "COMPUTER_SANDBOX_IDLE_AFTER") ?? "30m", + ), + allowPrivateHosts, + ...(computerToken ? { token: computerToken } : {}), + ...(policy ? { policy } : {}), + }; + } + const supervisorUrl = url(environment, "COMPUTER_SUPERVISOR_URL"); if (supervisorUrl) { const supervisorToken = optional(environment, "SUPERVISOR_TOKEN"); diff --git a/server/src/db/schema/index.ts b/server/src/db/schema/index.ts index b924af64..68ee5d83 100644 --- a/server/src/db/schema/index.ts +++ b/server/src/db/schema/index.ts @@ -3,5 +3,6 @@ export * from "./components"; export * from "./computer"; export * from "./core"; -export * from "./plugins"; export * from "./coworker"; +export * from "./plugins"; +export * from "./work"; diff --git a/server/src/db/schema/work.ts b/server/src/db/schema/work.ts new file mode 100644 index 00000000..31cd9a0b --- /dev/null +++ b/server/src/db/schema/work.ts @@ -0,0 +1,79 @@ +import { + index, + integer, + pgTable, + primaryKey, + text, + timestamp, +} from "drizzle-orm/pg-core"; +import { jsonb } from "./json"; + +const createdAt = () => + timestamp("created_at", { withTimezone: true }).notNull().defaultNow(); +const updatedAt = () => + timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(); + +/** + * Durable work, claimed by whichever replica gets there first, leased so a dead one's work comes + * back. + * + * ONE MECHANISM, THREE FEATURES. Suspending idle computers needs it, scheduled routines need it, and + * a hop from one Bot to another needs it. Written once with all three in view rather than three + * times slightly differently, because the parts that are easy to get wrong are the same every time: + * who owns an item, what happens when the owner dies, and whether a recovery can run something + * twice. + * + * POSTGRES, NOT A QUEUE. It is already the thing every replica shares, and `for update skip locked` + * is exactly this problem: each replica takes rows nobody else holds, no coordinator, no leader + * election, no single point of failure, and adding a replica adds throughput rather than contention. + * + * NOT `setInterval`. The audit-retention sweep uses one and is safe only because deleting old rows + * twice is the same as deleting them once. Anything that spends money, calls a tool or posts a + * message is not that: fired by every replica it happens N times, and on a cluster that is Tuesday. + */ +export const workItems = pgTable( + "work_items", + { + /** What kind of work. `computer.suspend` today; routines and bot-to-bot hops later. */ + kind: text("kind").notNull(), + /** + * What the work is about, unique within its kind. + * + * The Bot id for a computer to suspend, and for a routine the routine and the minute it was due, + * because IDEMPOTENCE LIVES HERE. A routine due at 07:00 must run once even if three replicas + * wake together and a lease is reclaimed mid-flight; making the key carry the scheduled time is + * what turns "fire it again" into "insert that already exists" rather than a second run. Without + * it every recovery path is also a duplicate-run path. + */ + key: text("key").notNull(), + /** When this becomes eligible. A claim never sees an item before its time. */ + runAt: timestamp("run_at", { withTimezone: true }).notNull().defaultNow(), + /** + * Which replica holds it, and until when. + * + * Null owner means nobody has it. A lease in the past means whoever had it stopped renewing, and + * the row is free again: that is the whole recovery story, and it needs no process to notice a + * death, only the next claim to look at the clock. + */ + claimedBy: text("claimed_by"), + leaseUntil: timestamp("lease_until", { withTimezone: true }), + /** + * How many times this has been handed out. + * + * A RECLAIMED ITEM IS NOT A FRESH ONE, and the difference matters enough to count rather than + * infer. An item on its first attempt has certainly not run; one on its second may already have + * called a tool and spent money before its owner died. Whatever picks it up has to be able to + * tell those apart, so it is a number here rather than a state folded into failure. + */ + attempts: integer("attempts").notNull().default(0), + /** Anything the work needs that is not in the key. */ + payload: jsonb("payload").notNull().default({}), + createdAt: createdAt(), + updatedAt: updatedAt(), + }, + (table) => [ + primaryKey({ columns: [table.kind, table.key] }), + // The claim's own query: due, unclaimed or expired, oldest first. + index("work_items_claimable_idx").on(table.kind, table.runAt), + ], +); diff --git a/server/src/work/culler.ts b/server/src/work/culler.ts new file mode 100644 index 00000000..eca55f36 --- /dev/null +++ b/server/src/work/culler.ts @@ -0,0 +1,174 @@ +/** + * Suspending computers nobody is using. + * + * A `Sandbox` has `shutdownTime`, an absolute expiry, which is not the question anybody is asking: + * "nobody has touched this for thirty minutes" is. So this asks that one, and it has to survive the + * replica that started it, which is why the work is claimed and leased out of PostgreSQL rather than + * held in a timer. + * + * NOTHING HERE TOUCHES A BROWSER. Deciding whether a computer is idle by dialling it would wake the + * computer, so every idle Bot anything asked about would come back up and the bill would never fall. + * That is the known, invisible way to lose scale-to-zero: everything works, nothing ever suspends. + * Idleness is read from the audit trail, which is a record of what a Bot did rather than a question + * put to the thing that did it. + */ +import { and, inArray, like, sql } from "drizzle-orm"; +import type { ComputerProvider } from "../computer/provider"; +import type { Database } from "../db/client"; +import { auditEvents } from "../db/schema"; +import type { WorkQueue } from "./queue"; + +export const CULL_KIND = "computer.suspend"; + +export type CullerOptions = { + database: Database; + queue: WorkQueue; + provider: ComputerProvider; + /** A computer untouched for this long is idle. */ + idleAfterMs: number; + /** Who this replica is, for the lease. */ + owner: string; + leaseMs?: number; + now?: () => Date; +}; + +export type CullReport = { + considered: number; + suspended: string[]; + skipped: { botId: string; reason: string }[]; +}; + +/** + * Which Bots have a computer running, and when each was last asked to do anything. + * + * The audit trail is the source: every acting call writes a row before the computer is touched, so + * "last used" is already recorded, server-side, and survives a restart. A computer that has never + * acted has no row and reads as idle since it started, which is the answer wanted. + */ +async function lastActedAt( + database: Database, + botIds: string[], +): Promise> { + if (botIds.length === 0) return new Map(); + /* + * The Bot is in the payload rather than in a column, so the grouping key is an expression. Written + * through the query builder rather than as one raw string, because a list parameter has to be + * bound as a list: handed to `= any($1)` as a JavaScript array it arrives as one opaque value and + * the query fails rather than matching nothing, which at least says so. + */ + const bot = sql`${auditEvents.payload}->>'bot'`; + const rows = await database + .select({ bot, last: sql`max(${auditEvents.createdAt})` }) + .from(auditEvents) + .where(and(like(auditEvents.eventType, "computer.%"), inArray(bot, botIds))) + .groupBy(bot); + + return new Map( + rows + .filter((row) => row.bot) + .map((row) => [row.bot, new Date(row.last)] as const), + ); +} + +/** + * Offer every idle computer for suspension. + * + * Offering rather than suspending: the decision and the act are separated so that whichever replica + * runs this does not have to be the one that carries it out, and so a suspension that fails halfway + * is retried by whoever picks the item up next rather than lost with the process that noticed. + */ +export async function offerIdleComputers( + options: CullerOptions, +): Promise<{ offered: string[] }> { + const now = options.now?.() ?? new Date(); + const computers = await options.provider.list(); + const running = computers.filter((computer) => computer.status === "running"); + const used = await lastActedAt( + options.database, + running.map((computer) => computer.botId), + ); + + const offered: string[] = []; + for (const computer of running) { + const since = + used.get(computer.botId) ?? + (computer.startedAt ? new Date(computer.startedAt) : undefined); + // No row and no start time means nothing is known about it, and suspending on no evidence is + // how somebody's session disappears mid-task. Left alone. + if (!since) continue; + if (now.getTime() - since.getTime() < options.idleAfterMs) continue; + await options.queue.offer({ + kind: CULL_KIND, + key: computer.botId, + payload: { botId: computer.botId, idleSince: since.toISOString() }, + }); + offered.push(computer.botId); + } + return { offered }; +} + +/** + * Carry out whatever suspensions this replica can claim. + * + * Re-checked at the moment of acting, because the decision was made by another replica at another + * time and a person may have started working in between. Suspending a computer somebody is using is + * worse than leaving an idle one running for another five minutes. + */ +export async function suspendClaimedComputers( + options: CullerOptions, +): Promise { + const leaseMs = options.leaseMs ?? 60_000; + const claimed = await options.queue.claim({ + kind: CULL_KIND, + owner: options.owner, + leaseMs, + limit: 20, + }); + + const report: CullReport = { + considered: claimed.length, + suspended: [], + skipped: [], + }; + + for (const item of claimed) { + const botId = String(item.payload.botId ?? item.key); + try { + const now = options.now?.() ?? new Date(); + const used = await lastActedAt(options.database, [botId]); + const since = used.get(botId); + if (since && now.getTime() - since.getTime() < options.idleAfterMs) { + // Somebody came back. Drop the item rather than releasing it, because the next sweep will + // offer it again if it goes quiet, and a released one would just be reclaimed and re-checked. + await options.queue.finish({ kind: CULL_KIND, key: item.key }); + report.skipped.push({ + botId, + reason: "used again before it was suspended", + }); + continue; + } + + await options.provider.stop(botId); + await options.queue.finish({ kind: CULL_KIND, key: item.key }); + report.suspended.push(botId); + } catch (error) { + /* + * Released rather than dropped, and pushed out rather than retried immediately: a cluster that + * refused this once will probably refuse it again in the next second, and a computer left + * running costs money rather than losing anything. + */ + await options.queue.release({ + kind: CULL_KIND, + key: item.key, + delayMs: 5 * 60_000, + }); + report.skipped.push({ + botId, + reason: + error instanceof Error ? error.message : "could not be suspended", + }); + } + } + + return report; +} diff --git a/server/src/work/queue.ts b/server/src/work/queue.ts new file mode 100644 index 00000000..ecec5619 --- /dev/null +++ b/server/src/work/queue.ts @@ -0,0 +1,180 @@ +/** + * Claiming durable work, so several replicas can share it without a coordinator. + * + * `select ... for update skip locked` inside a transaction: each replica takes rows nobody else + * holds and never waits behind another's. No leader election, no single point of failure, and a + * replica added is throughput added rather than contention added. + * + * A claim carries a lease. While the work runs its owner renews; one that stops being renewed is + * free again the moment anything looks, so recovery needs no process to notice a death, only the + * next claim to read the clock. + */ +import { and, eq, sql } from "drizzle-orm"; +import type { Database } from "../db/client"; +import { workItems } from "../db/schema"; + +export type WorkItem = { + kind: string; + key: string; + payload: Record; + /** + * How many times this has been handed out, including now. + * + * ONE MEANS IT HAS CERTAINLY NOT RUN. More than one means a previous owner stopped renewing, and + * may already have called a tool or spent money before it did. A caller that cannot tell those + * apart cannot safely retry anything with an outside effect, so this is a number rather than a + * state folded into failure. + */ + attempts: number; +}; + +export type WorkQueue = { + /** Put work on the queue, or leave what is there. Idempotent on (kind, key). */ + offer: (item: { + kind: string; + key: string; + payload?: Record; + runAt?: Date; + }) => Promise; + /** Take up to `limit` due items, leased to `owner`. */ + claim: (input: { + kind: string; + owner: string; + leaseMs: number; + limit?: number; + }) => Promise; + /** Keep a claim alive while the work runs. False means it was already taken away. */ + renew: (input: { + kind: string; + key: string; + owner: string; + leaseMs: number; + }) => Promise; + /** Done. The row goes, so the same key can be offered again later. */ + finish: (input: { kind: string; key: string }) => Promise; + /** Not done, and worth another go after `delayMs`. */ + release: (input: { + kind: string; + key: string; + delayMs: number; + }) => Promise; +}; + +export function createWorkQueue(database: Database): WorkQueue { + return { + async offer({ kind, key, payload = {}, runAt }) { + await database + .insert(workItems) + .values({ kind, key, payload, ...(runAt ? { runAt } : {}) }) + /* + * Nothing on conflict, deliberately. + * + * The key is the identity of the work, so a second offer of the same thing is the same + * thing, not a new one. For a routine the key carries the minute it was due, which is what + * makes "three replicas woke at 07:00" produce one run instead of three. + */ + .onConflictDoNothing(); + }, + + async claim({ kind, owner, leaseMs, limit = 1 }) { + return database.transaction(async (transaction) => { + /* + * `skip locked` is what makes this concurrent rather than merely correct. Without it a + * second replica blocks on the first replica's rows and the queue serialises; with it, it + * walks past them and takes the next free ones. + */ + const due = await transaction.execute(sql` + select "kind", "key" + from "work_items" + where "kind" = ${kind} + and "run_at" <= now() + and ("lease_until" is null or "lease_until" <= now()) + order by "run_at" asc + limit ${limit} + for update skip locked + `); + + const rows = ( + Array.isArray(due) ? due : ((due as { rows?: unknown[] })?.rows ?? []) + ) as { + kind: string; + key: string; + }[]; + if (rows.length === 0) return []; + + const claimed: WorkItem[] = []; + for (const row of rows) { + const [updated] = await transaction + .update(workItems) + .set({ + claimedBy: owner, + leaseUntil: new Date(Date.now() + leaseMs), + attempts: sql`${workItems.attempts} + 1`, + updatedAt: new Date(), + }) + .where( + and(eq(workItems.kind, row.kind), eq(workItems.key, row.key)), + ) + .returning({ + kind: workItems.kind, + key: workItems.key, + payload: workItems.payload, + attempts: workItems.attempts, + }); + if (updated) { + claimed.push({ + kind: updated.kind, + key: updated.key, + payload: (updated.payload ?? {}) as Record, + attempts: updated.attempts, + }); + } + } + return claimed; + }); + }, + + async renew({ kind, key, owner, leaseMs }) { + const [renewed] = await database + .update(workItems) + .set({ + leaseUntil: new Date(Date.now() + leaseMs), + updatedAt: new Date(), + }) + /* + * Only while still ours. A lease that expired and was taken by somebody else must not be + * renewed back out from under them, which would put two replicas on one item believing they + * each held it. + */ + .where( + and( + eq(workItems.kind, kind), + eq(workItems.key, key), + eq(workItems.claimedBy, owner), + ), + ) + .returning({ key: workItems.key }); + return Boolean(renewed); + }, + + async finish({ kind, key }) { + await database + .delete(workItems) + .where(and(eq(workItems.kind, kind), eq(workItems.key, key))); + }, + + async release({ kind, key, delayMs }) { + // Freed and pushed out, rather than deleted: the work still wants doing, just not immediately + // and not by whoever just gave up on it. + await database + .update(workItems) + .set({ + claimedBy: null, + leaseUntil: null, + runAt: new Date(Date.now() + delayMs), + updatedAt: new Date(), + }) + .where(and(eq(workItems.kind, kind), eq(workItems.key, key))); + }, + }; +} diff --git a/server/tests/computer-culler.integration.test.ts b/server/tests/computer-culler.integration.test.ts new file mode 100644 index 00000000..5ad05747 --- /dev/null +++ b/server/tests/computer-culler.integration.test.ts @@ -0,0 +1,208 @@ +import { afterAll, beforeEach, describe, expect, test } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import type { + ComputerLocation, + ComputerProvider, +} from "../src/computer/provider"; +import { createDatabase } from "../src/db/client"; +import { auditEvents, workItems } from "../src/db/schema"; +import { + CULL_KIND, + offerIdleComputers, + suspendClaimedComputers, +} from "../src/work/culler"; +import { createWorkQueue } from "../src/work/queue"; +import { TEST_POOL } from "./support/database"; + +/** + * Spinning a computer down when nobody is using it, which is the whole reason the fleet does not + * cost a hundred idle browsers. + * + * The interesting cases are the ones where it must NOT act: a computer somebody just used, and a + * computer nothing is known about. Suspending either takes a person's session away mid-task, and + * both are easy to get wrong in a way no error ever reports. + */ +const database = createDatabase(TEST_POOL); +const queue = createWorkQueue(database); +const suite = randomUUID().slice(0, 8); +const botOf = (name: string) => `cull-${suite}-${name}`; + +function providerWith(computers: ComputerLocation[]) { + const stopped: string[] = []; + const provider: ComputerProvider = { + name: "fake", + isolation: "per-bot", + locate: async () => "http://unused", + status: async (botId) => ({ botId, state: "ready" }), + stop: async (botId) => { + stopped.push(botId); + return { wasRunning: true }; + }, + reset: async () => ({ cleared: false }), + list: async () => computers, + }; + return { provider, stopped }; +} + +const ran = (botId: string, when: Date) => ({ + eventType: "computer.action_allowed", + targetType: "computer", + targetId: botId, + payload: { bot: botId, action: "computer_navigate" }, + createdAt: when, +}); + +afterAll(async () => { + await database.delete(workItems).where(eq(workItems.kind, CULL_KIND)); + await database.$client.end({ timeout: 5 }); +}); + +beforeEach(async () => { + await database.delete(workItems).where(eq(workItems.kind, CULL_KIND)); +}); + +const idleAfterMs = 30 * 60_000; +const now = () => new Date("2026-08-24T12:00:00Z"); +const minutesAgo = (minutes: number) => + new Date(now().getTime() - minutes * 60_000); + +describe("suspending computers nobody is using", () => { + test("a computer idle longer than the threshold is offered, and then suspended", async () => { + const botId = botOf("idle"); + await database.insert(auditEvents).values(ran(botId, minutesAgo(45))); + const { provider, stopped } = providerWith([ + { botId, status: "running", url: "http://c" }, + ]); + const options = { + database, + queue, + provider, + idleAfterMs, + owner: "replica-1", + now, + }; + + expect((await offerIdleComputers(options)).offered).toEqual([botId]); + const report = await suspendClaimedComputers(options); + + expect(report.suspended).toEqual([botId]); + expect(stopped).toEqual([botId]); + }); + + test("a computer used a minute ago is left alone", async () => { + const botId = botOf("busy"); + await database.insert(auditEvents).values(ran(botId, minutesAgo(1))); + const { provider, stopped } = providerWith([ + { botId, status: "running", url: "http://c" }, + ]); + + const offered = await offerIdleComputers({ + database, + queue, + provider, + idleAfterMs, + owner: "replica-1", + now, + }); + + expect(offered.offered).toEqual([]); + expect(stopped).toEqual([]); + }); + + /* + * The race the lease exists for. One replica decides a computer is idle; before anything acts on + * that, the person comes back. Suspending them mid-task is worse than paying for another five + * minutes of an idle browser, so the decision is re-checked at the moment of acting. + */ + test("a computer used after being offered is not suspended", async () => { + const botId = botOf("returned"); + await database.insert(auditEvents).values(ran(botId, minutesAgo(45))); + const { provider, stopped } = providerWith([ + { botId, status: "running", url: "http://c" }, + ]); + const options = { + database, + queue, + provider, + idleAfterMs, + owner: "replica-1", + now, + }; + + await offerIdleComputers(options); + // They came back while the item sat on the queue. + await database.insert(auditEvents).values(ran(botId, minutesAgo(2))); + const report = await suspendClaimedComputers(options); + + expect(stopped).toEqual([]); + expect(report.skipped[0]?.reason).toContain("used again"); + }); + + test("a computer nothing is known about is left alone", async () => { + // No audit row and no start time. Suspending on no evidence is how a session disappears. + const botId = botOf("unknown"); + const { provider, stopped } = providerWith([ + { botId, status: "running", url: "http://c" }, + ]); + + const offered = await offerIdleComputers({ + database, + queue, + provider, + idleAfterMs, + owner: "replica-1", + now, + }); + + expect(offered.offered).toEqual([]); + expect(stopped).toEqual([]); + }); + + test("a computer already stopped is not offered again", async () => { + const botId = botOf("stopped"); + await database.insert(auditEvents).values(ran(botId, minutesAgo(90))); + const { provider } = providerWith([ + { botId, status: "stopped", url: "http://c" }, + ]); + + expect( + ( + await offerIdleComputers({ + database, + queue, + provider, + idleAfterMs, + owner: "replica-1", + now, + }) + ).offered, + ).toEqual([]); + }); + + test("two replicas culling together suspend each computer once", async () => { + const ids = ["a", "b", "c", "d"].map(botOf); + for (const botId of ids) { + await database.insert(auditEvents).values(ran(botId, minutesAgo(60))); + } + const { provider, stopped } = providerWith( + ids.map((botId) => ({ + botId, + status: "running" as const, + url: "http://c", + })), + ); + const base = { database, queue, provider, idleAfterMs, now }; + + await offerIdleComputers({ ...base, owner: "replica-1" }); + const [first, second] = await Promise.all([ + suspendClaimedComputers({ ...base, owner: "replica-1" }), + suspendClaimedComputers({ ...base, owner: "replica-2" }), + ]); + + const all = [...first.suspended, ...second.suspended]; + expect(all.sort()).toEqual([...ids].sort()); + // Each exactly once, which is the point of claiming rather than sweeping. + expect(new Set(stopped).size).toBe(stopped.length); + }); +}); diff --git a/server/tests/work-queue.integration.test.ts b/server/tests/work-queue.integration.test.ts new file mode 100644 index 00000000..506f145a --- /dev/null +++ b/server/tests/work-queue.integration.test.ts @@ -0,0 +1,187 @@ +import { afterAll, beforeEach, describe, expect, test } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { and, eq, inArray } from "drizzle-orm"; +import { createDatabase } from "../src/db/client"; +import { workItems } from "../src/db/schema"; +import { createWorkQueue } from "../src/work/queue"; +import { TEST_POOL } from "./support/database"; + +/** + * The one mechanism suspending idle computers, running routines and handing work between Bots all + * need, driven against a real PostgreSQL rather than a fake. + * + * A fake cannot answer the only question worth asking here. `for update skip locked` is a promise + * the database makes about two transactions racing, and a stub that returns rows in order would pass + * every test below while the real thing handed one item to two replicas. + */ +const database = createDatabase(TEST_POOL); +const queue = createWorkQueue(database); +const kind = `test.${randomUUID().slice(0, 8)}`; + +afterAll(async () => { + await database.delete(workItems).where(eq(workItems.kind, kind)); + await database.$client.end({ timeout: 5 }); +}); + +beforeEach(async () => { + await database.delete(workItems).where(eq(workItems.kind, kind)); +}); + +describe("claiming durable work", () => { + test("one replica takes a due item, and it comes back with what it is about", async () => { + await queue.offer({ kind, key: "bot-a", payload: { botId: "bot-a" } }); + + const claimed = await queue.claim({ + kind, + owner: "replica-1", + leaseMs: 30_000, + }); + + expect(claimed).toHaveLength(1); + expect(claimed[0]?.key).toBe("bot-a"); + expect(claimed[0]?.payload).toEqual({ botId: "bot-a" }); + // First time out, so whatever runs this knows it has certainly not run before. + expect(claimed[0]?.attempts).toBe(1); + }); + + test("a second replica does not get an item the first is holding", async () => { + await queue.offer({ kind, key: "bot-a" }); + + const first = await queue.claim({ + kind, + owner: "replica-1", + leaseMs: 30_000, + }); + const second = await queue.claim({ + kind, + owner: "replica-2", + leaseMs: 30_000, + }); + + expect(first).toHaveLength(1); + expect(second).toHaveLength(0); + }); + + /* + * THE TEST THIS FILE EXISTS FOR. + * + * Ten replicas reaching for ten items at the same moment must between them take each item once. + * Anything less than `skip locked` fails here: plain `for update` serialises and one transaction + * waits behind another, and no locking at all hands the same row to several claimants, which for a + * routine is the same run billed N times. + */ + test("ten replicas racing for ten items take each of them exactly once", async () => { + const keys = Array.from({ length: 10 }, (_, index) => `bot-${index}`); + for (const key of keys) await queue.offer({ kind, key }); + + const results = await Promise.all( + Array.from({ length: 10 }, (_, index) => + queue.claim({ + kind, + owner: `replica-${index}`, + leaseMs: 30_000, + limit: 3, + }), + ), + ); + + const taken = results.flat().map((item) => item.key); + expect(taken).toHaveLength(10); + expect(new Set(taken).size).toBe(10); + }); + + test("a lease that stopped being renewed comes back to whoever asks next", async () => { + await queue.offer({ kind, key: "bot-a" }); + // Claimed by a replica that then dies: nothing renews, and the lease is already in the past. + await queue.claim({ kind, owner: "replica-1", leaseMs: -1 }); + + const recovered = await queue.claim({ + kind, + owner: "replica-2", + leaseMs: 30_000, + }); + + expect(recovered).toHaveLength(1); + /* + * Second time out, which is the number that matters. Whatever picks this up has to be able to + * tell "this never started" from "this started and we lost the process", because the second may + * already have called a tool and spent money. + */ + expect(recovered[0]?.attempts).toBe(2); + }); + + test("renewing keeps a claim, and cannot steal one back after it was lost", async () => { + await queue.offer({ kind, key: "bot-a" }); + await queue.claim({ kind, owner: "replica-1", leaseMs: -1 }); + await queue.claim({ kind, owner: "replica-2", leaseMs: 30_000 }); + + // Replica 1 wakes up and tries to keep a claim that is no longer its own. + const stale = await queue.renew({ + kind, + key: "bot-a", + owner: "replica-1", + leaseMs: 30_000, + }); + const current = await queue.renew({ + kind, + key: "bot-a", + owner: "replica-2", + leaseMs: 30_000, + }); + + expect(stale).toBe(false); + expect(current).toBe(true); + }); + + test("offering the same work twice leaves one item", async () => { + /* + * Idempotence, which is where every recovery path stops being a duplicate-run path. A routine + * due at 07:00 is offered by every replica that wakes; the key carries the minute, so they are + * all offering the same thing. + */ + await queue.offer({ kind, key: "routine:daily:2026-08-24T07:00" }); + await queue.offer({ kind, key: "routine:daily:2026-08-24T07:00" }); + await queue.offer({ kind, key: "routine:daily:2026-08-24T07:00" }); + + const rows = await database + .select({ key: workItems.key }) + .from(workItems) + .where(eq(workItems.kind, kind)); + expect(rows).toHaveLength(1); + }); + + test("an item that is not due yet is not claimed", async () => { + await queue.offer({ + kind, + key: "later", + runAt: new Date(Date.now() + 60_000), + }); + + expect( + await queue.claim({ kind, owner: "replica-1", leaseMs: 30_000 }), + ).toHaveLength(0); + }); + + test("finishing removes the item, so the same key can be offered again", async () => { + await queue.offer({ kind, key: "bot-a" }); + await queue.claim({ kind, owner: "replica-1", leaseMs: 30_000 }); + await queue.finish({ kind, key: "bot-a" }); + + const rows = await database + .select({ key: workItems.key }) + .from(workItems) + .where(and(eq(workItems.kind, kind), inArray(workItems.key, ["bot-a"]))); + expect(rows).toHaveLength(0); + }); + + test("releasing frees the item and holds it back for a while", async () => { + await queue.offer({ kind, key: "bot-a" }); + await queue.claim({ kind, owner: "replica-1", leaseMs: 30_000 }); + await queue.release({ kind, key: "bot-a", delayMs: 60_000 }); + + // Free, but not yet due, so nobody picks it straight back up and spins on it. + expect( + await queue.claim({ kind, owner: "replica-2", leaseMs: 30_000 }), + ).toHaveLength(0); + }); +}); From e9d0c4babc3f2b0999c1d8f99a6d1e44a445758e Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 24 Aug 2026 17:36:12 -0700 Subject: [PATCH 3/9] Refuse a sandbox install on a cluster that cannot make one computers.mode: sandbox creates Sandbox objects, which exist only once the agent-sandbox controller is installed. Without it the install succeeds, every pod is healthy, and the deployment looks finished right up until the first Bot asks for a browser and the API server answers 404. That is the worst moment to learn it. The check reads the cluster rather than a value somebody has to remember to set, and the message carries the one command that fixes it. Proven both ways: refused on a cluster with no CRD, installs on the EKS cluster that has one. Also from driving it on real EKS: lost+found was listed as a Bot, because an EBS volume is ext4 and arrives with that directory, which a bind mount never does. The allow-list that stops a hostile id becoming a path answers the other half of the question too. The migration Job named a ServiceAccount that does not exist yet, since a pre-install hook runs before the chart's own resources. It talks to a database and never to the cluster, so it needs no account at all. The API pod gets a cluster token only in sandbox mode, the pods roll when the computer template changes, the Sandbox asks for a Service so it has an address that survives a resume, and the cluster CA is actually used when talking to the API server. --- CHANGELOG.md | 5 + agent-computer/src/profiles.ts | 20 +++- agent-computer/tests/profile-listing.test.ts | 45 ++++++++ charts/openbot/README.md | 32 +++++- charts/openbot/templates/_helpers.tpl | 83 ++++++++++++++ .../templates/computer/culler-cronjob.yaml | 9 ++ .../templates/computer/pod-template.yaml | 24 ++++ .../templates/computer/sandbox-rbac.yaml | 3 +- .../templates/computer/sandbox-template.yaml | 104 ++---------------- .../openbot/templates/computer/warmpool.yaml | 2 +- charts/openbot/templates/migrations/job.yaml | 12 +- .../openbot/templates/server/deployment.yaml | 26 ++++- charts/openbot/templates/server/service.yaml | 4 + .../templates/server/serviceaccount.yaml | 2 +- charts/openbot/templates/validation.yaml | 29 +++++ charts/openbot/values.yaml | 27 +++++ server/src/computer/provider.ts | 26 ++--- server/src/computer/sandbox.ts | 54 ++++++++- server/src/config.ts | 5 + 19 files changed, 390 insertions(+), 122 deletions(-) create mode 100644 agent-computer/tests/profile-listing.test.ts create mode 100644 charts/openbot/templates/computer/pod-template.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index e60fab07..91e2eba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,11 @@ isolated, stateful, singleton pod with a stable identity and persistent storage. field, and it keeps the volumes, so a computer comes back with its logins rather than signed out of everything. `shared` stays the default and needs nothing installed in the cluster. +**A cluster with no controller is refused at install.** `computers.mode: sandbox` needs the +agent-sandbox CRD, and without it the install succeeds, every pod is healthy, and the deployment +looks finished until the first Bot asks for a browser. The chart reads the cluster and refuses, +naming the one command that fixes it. + **What decides a computer is idle is the audit trail, not the browser.** Asking the browser would wake it, so every computer anything asked about would come back up and the bill would never fall. diff --git a/agent-computer/src/profiles.ts b/agent-computer/src/profiles.ts index 420931ee..5f1b1fd4 100644 --- a/agent-computer/src/profiles.ts +++ b/agent-computer/src/profiles.ts @@ -36,7 +36,7 @@ import { readdir, rm } from "node:fs/promises"; import { join } from "node:path"; import { type BrowserContext, chromium, type Page } from "playwright"; -import { profileDirectoryFor } from "./bot-id"; +import { isPlainBotId, profileDirectoryFor } from "./bot-id"; import { chooseEvictions, chooseIdle } from "./browser-eviction"; import { egressFor, egressLabel } from "./egress"; import { numberFromEnv } from "./env"; @@ -370,7 +370,23 @@ export function createProfiles(root: string) { ); return [ ...new Set([ - ...onDisk.filter((e) => e.isDirectory()).map((e) => e.name), + ...onDisk + .filter((e) => e.isDirectory()) + /* + * ONLY DIRECTORIES THIS CODE COULD HAVE MADE. + * + * The root is a mounted volume, and a volume is not an empty directory: a real disk + * formatted ext4 arrives with `lost+found` already in it, so on a cloud the fleet page + * listed a Bot by that name, offered to reset it, and nobody could say where it came + * from. Never seen locally, because a bind mount and kind's local-path volumes have no + * such directory, which is exactly the shape of bug that ships. + * + * `isPlainBotId` is the same allow-list that stops a hostile id becoming a path, used + * here for the other half of the question: an entry it would refuse to create is not one + * of ours to list. `lost+found` fails it on the `+`. + */ + .filter((e) => isPlainBotId(e.name)) + .map((e) => e.name), ...live.keys(), ]), ].sort(); diff --git a/agent-computer/tests/profile-listing.test.ts b/agent-computer/tests/profile-listing.test.ts new file mode 100644 index 00000000..0c4e2ff2 --- /dev/null +++ b/agent-computer/tests/profile-listing.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, readdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { isPlainBotId } from "../src/bot-id"; + +/** + * Which directories under the profiles root are Bots. + * + * The root is a mounted volume, and a volume is not an empty directory. A real disk formatted ext4 + * arrives with `lost+found` already in it, so on a cloud the fleet page listed a Bot by that name and + * offered to reset it. Never seen locally, because a bind mount and kind's local-path volumes have no + * such directory: the bug appears only on the deployment shape the feature is for. + * + * The rule is the one that already exists. `isPlainBotId` decides what may become a profile path, so + * an entry it would refuse to create is not one of ours to list. + */ +async function knownIn(root: string): Promise { + const onDisk = await readdir(root, { withFileTypes: true }).catch(() => []); + return onDisk + .filter((entry) => entry.isDirectory()) + .filter((entry) => isPlainBotId(entry.name)) + .map((entry) => entry.name) + .sort(); +} + +describe("listing the Bots that have a computer", () => { + test("lists Bot profiles and ignores what the filesystem put there", async () => { + const root = await mkdtemp(join(tmpdir(), "profiles-")); + await mkdir(join(root, "knowledge")); + await mkdir(join(root, "risk-analyst")); + // What an ext4 volume brings with it, which is the whole reason this test exists. + await mkdir(join(root, "lost+found")); + // A file is not a computer either. + await writeFile(join(root, "notes.txt"), ""); + + expect(await knownIn(root)).toEqual(["knowledge", "risk-analyst"]); + }); + + test("the name a real volume arrives with is not a usable Bot id", () => { + // Stated directly, because this is the property the filter leans on. + expect(isPlainBotId("lost+found")).toBe(false); + expect(isPlainBotId("knowledge")).toBe(true); + }); +}); diff --git a/charts/openbot/README.md b/charts/openbot/README.md index 89a837ec..9aaac36e 100644 --- a/charts/openbot/README.md +++ b/charts/openbot/README.md @@ -35,14 +35,30 @@ works on its own. A replica must not carry one: a browser is a few hundred megab Bot's logins, so scaling the API would scale those with it. `server.embeddedComputer` is off here, and asking for it with more than one replica is refused at install time. -## Your own database +## Your own database, which is what a real deployment uses ```sh --set postgresql.enabled=false \ --set database.existingSecret=openbot-database # key: database-url ``` -Setting both a bundled database and a URL is refused, rather than one of them silently winning. +`postgresql.enabled` is **off by default and not production-grade**. A database on a pod goes away +when the pod does: a rollout, a node drain or an eviction is a restart, and while the volume survives, +nothing about that shape gives you backups, failover or point-in-time recovery. It is there so +somebody can try OpenBot in one command. + +Point it at RDS, Cloud SQL, Azure Database or your own server, and keep the URL in a Secret rather +than in a values file. Setting both a bundled database and a URL is refused, rather than one of them +silently winning. + +**Put `?sslmode=require` on the URL.** Every managed database refuses an unencrypted connection: +RDS has `rds.force_ssl` on by default, and Cloud SQL and Azure Database do the same. Without it the +migration fails with `no pg_hba.conf entry for host ... no encryption`, which names the host and the +user and not the actual problem. + +**The migrating role has to be able to create and drop the `vector` extension.** The first migration +creates it and a later one drops it again. On a managed database, create it once as the +administrative role; `CREATE EXTENSION IF NOT EXISTS` then passes for an ordinary user. ## The four targets @@ -127,6 +143,18 @@ that mode, which is stated on the fleet page rather than hidden. workload: an isolated, stateful, singleton pod with a stable identity and persistent storage. Suspending is `operatingMode: Suspended`, which terminates the pod and keeps the volumes. +**That controller is not installed by this chart, and the chart refuses to install without it.** +The check reads the cluster, so it is a real answer rather than a value somebody has to remember: + +```sh +kubectl apply --server-side -f \ + https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v0.5.6/sandbox-with-extensions.yaml +``` + +Without that refusal the install succeeds, every pod is healthy, and the deployment looks finished +until the first Bot asks for a browser and the API server answers 404. Rendering offline? Pass +`--api-versions agents.x-k8s.io/v1beta1/Sandbox`. + **What decides that a computer is idle** is the audit trail, not the browser. Asking the browser would wake it, so every computer anything asked about would come back up and the bill would never fall. That is the known, invisible way to lose scale-to-zero: everything works, nothing suspends. diff --git a/charts/openbot/templates/_helpers.tpl b/charts/openbot/templates/_helpers.tpl index fe458c5e..d973cf2a 100644 --- a/charts/openbot/templates/_helpers.tpl +++ b/charts/openbot/templates/_helpers.tpl @@ -171,6 +171,8 @@ and in whatever holds the release, which is not where `KEY_ENCRYPTION_KEY` belon value: {{ default .Release.Namespace .Values.computers.sandbox.namespace | quote }} - name: COMPUTER_SANDBOX_IDLE_AFTER value: {{ .Values.computers.sandbox.idleAfter | quote }} +- name: COMPUTER_SANDBOX_TEMPLATE_FILE + value: /etc/openbot/sandbox-template.json {{- end }} - name: INTELLIGENCE_API_URL value: {{ .Values.config.intelligence.apiUrl | quote }} @@ -278,3 +280,84 @@ podAntiAffinity: {{ include "openbot.componentSelectorLabels" (dict "root" $root "component" $component) | indent 12 }} {{- end -}} {{- end -}} + +{{/* +The pod and volumes every Bot's computer is cut from, as JSON. + +One definition, used by the ConfigMap the server reads and by the SandboxTemplate a warm pool cuts +from, so a pre-warmed computer and one created on demand cannot drift into being different things. +*/}} +{{- define "openbot.sandboxPodTemplate" -}} +{{- $spec := dict + "podTemplate" (dict + "metadata" (dict "labels" (dict + "app.kubernetes.io/name" (include "openbot.name" .) + "app.kubernetes.io/instance" .Release.Name + "app.kubernetes.io/component" "computer")) + "spec" (dict + "terminationGracePeriodSeconds" 30 + "containers" (list (dict + "name" "computer" + "image" (include "openbot.image" .) + "imagePullPolicy" .Values.image.pullPolicy + "command" (list "/usr/local/bin/bun" "/app/agent-computer/src/index.ts") + "ports" (list (dict "name" "http" "containerPort" 4100)) + "env" (concat + (list + (dict "name" "PORT" "value" "4100") + (dict "name" "WORKSPACE_DIR" "value" "/workspace") + (dict "name" "PROFILES_DIR" "value" "/profiles") + (dict "name" "COMPUTER_TOKEN" "valueFrom" (dict "secretKeyRef" (dict + "name" (default (include "openbot.secretName" .) .Values.computers.existingTokenSecret) + "key" "computer-token")))) + .Values.computers.extraEnv) + "volumeMounts" (list + (dict "name" "profiles" "mountPath" "/profiles") + (dict "name" "workspace" "mountPath" "/workspace")) + "readinessProbe" (dict + "httpGet" (dict "path" "/health" "port" "http") + "periodSeconds" 10 + "failureThreshold" 6) + "resources" .Values.computers.resources)))) -}} +{{- $pod := index $spec "podTemplate" -}} +{{- $podSpec := index $pod "spec" -}} +{{- with .Values.computers.runtimeClassName }}{{- $_ := set $podSpec "runtimeClassName" . }}{{- end }} +{{- with .Values.imagePullSecrets }}{{- $_ := set $podSpec "imagePullSecrets" . }}{{- end }} +{{- with .Values.computers.nodeSelector }}{{- $_ := set $podSpec "nodeSelector" . }}{{- end }} +{{- with .Values.computers.tolerations }}{{- $_ := set $podSpec "tolerations" . }}{{- end }} +{{- $claim := dict + "accessModes" (list "ReadWriteOnce") + "resources" (dict "requests" (dict "storage" .Values.computers.persistence.profilesSize)) -}} +{{- $work := dict + "accessModes" (list "ReadWriteOnce") + "resources" (dict "requests" (dict "storage" .Values.computers.persistence.workspaceSize)) -}} +{{- with .Values.computers.persistence.storageClass }} +{{- $_ := set $claim "storageClassName" . }}{{- $_ := set $work "storageClassName" . }} +{{- end }} +{{- /* + A Service, which is the whole reason a computer has a stable address. + + Without it the controller creates the pod and reports no `serviceFQDN`, so the sandbox is Ready and + unreachable: `locate` waits for an address that is never coming and times out. A pod IP would be + the wrong answer anyway, because it changes on every resume, which is exactly what a suspended + computer does. +*/}} +{{- $_ := set $spec "service" true -}} +{{- $_ := set $spec "volumeClaimTemplates" (list + (dict "metadata" (dict "name" "profiles") "spec" $claim) + (dict "metadata" (dict "name" "workspace") "spec" $work)) -}} +{{ toPrettyJson $spec }} +{{- end -}} + +{{/* +Whether the API pod gets a Kubernetes token. + +FALSE UNLESS IT ACTUALLY NEEDS ONE. The API talks to a database and to Bots, not to the cluster, so a +mounted token is a credential sitting in a pod that has no use for it. `computers.mode: sandbox` is +the exception and the only one: there the server asks the API server to create, resume and suspend a +Sandbox per Bot, and without a token it fails on the first browser action with a missing file rather +than anything that names the cause. +*/}} +{{- define "openbot.automountToken" -}} +{{- or .Values.serviceAccount.automountServiceAccountToken (eq .Values.computers.mode "sandbox") -}} +{{- end -}} diff --git a/charts/openbot/templates/computer/culler-cronjob.yaml b/charts/openbot/templates/computer/culler-cronjob.yaml index 62e30d3f..7497a207 100644 --- a/charts/openbot/templates/computer/culler-cronjob.yaml +++ b/charts/openbot/templates/computer/culler-cronjob.yaml @@ -50,10 +50,19 @@ spec: env: {{ include "openbot.databaseUrlEnv" . | indent 16 }} {{ include "openbot.commonEnv" . | indent 16 }} + {{- /* The same shape of computer the server uses, so the two cannot disagree. */}} + volumeMounts: + - name: sandbox-template + mountPath: /etc/openbot + readOnly: true resources: requests: cpu: 50m memory: 128Mi limits: memory: 512Mi + volumes: + - name: sandbox-template + configMap: + name: {{ include "openbot.componentName" (dict "root" . "component" "computer-template") }} {{- end }} diff --git a/charts/openbot/templates/computer/pod-template.yaml b/charts/openbot/templates/computer/pod-template.yaml new file mode 100644 index 00000000..67e39a22 --- /dev/null +++ b/charts/openbot/templates/computer/pod-template.yaml @@ -0,0 +1,24 @@ +{{- if eq .Values.computers.mode "sandbox" }} +{{/* +What a Bot's computer looks like, handed to the server as a file. + +`Sandbox` takes its pod inline and has no template reference, so something has to give the server the +shape of a computer. It belongs here rather than in the code: which image a computer runs, what +volumes it keeps, and which RuntimeClass it uses are deployment decisions, and the three clouds do +not agree about the last one. + +MOUNTED, NOT FETCHED. A ConfigMap the server reads through the API would need a credential to read +ConfigMaps, and this needs no permission at all if the pod simply has the file. It also means the +template cannot change under a running server without a rollout, which is what the checksum +annotation on the Deployment already arranges. +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "openbot.componentName" (dict "root" . "component" "computer-template") }} + labels: +{{ include "openbot.componentLabels" (dict "root" . "component" "computer") | indent 4 }} +data: + sandbox-template.json: | +{{ include "openbot.sandboxPodTemplate" . | indent 4 }} +{{- end }} diff --git a/charts/openbot/templates/computer/sandbox-rbac.yaml b/charts/openbot/templates/computer/sandbox-rbac.yaml index 14e57a82..2bf66d53 100644 --- a/charts/openbot/templates/computer/sandbox-rbac.yaml +++ b/charts/openbot/templates/computer/sandbox-rbac.yaml @@ -23,7 +23,8 @@ rules: resources: ["sandboxes/status"] verbs: ["get"] {{- if .Values.computers.sandbox.warmPool.enabled }} - - apiGroups: ["agents.x-k8s.io"] + {{- /* The extension kinds are in their own API group, which a Role must name exactly. */}} + - apiGroups: ["extensions.agents.x-k8s.io"] resources: ["sandboxclaims"] verbs: ["get", "list", "create", "delete"] {{- end }} diff --git a/charts/openbot/templates/computer/sandbox-template.yaml b/charts/openbot/templates/computer/sandbox-template.yaml index d23a2418..90046594 100644 --- a/charts/openbot/templates/computer/sandbox-template.yaml +++ b/charts/openbot/templates/computer/sandbox-template.yaml @@ -1,13 +1,17 @@ -{{- if eq .Values.computers.mode "sandbox" }} +{{- if and (eq .Values.computers.mode "sandbox") .Values.computers.sandbox.warmPool.enabled }} {{- $ns := default .Release.Namespace .Values.computers.sandbox.namespace -}} {{/* -The pod every Bot's computer is cut from. +The template a warm pool cuts pre-warmed computers from. -Here rather than in the server, because what image a computer runs, what volumes it keeps and what -runtime class it uses are deployment decisions, and the three clouds disagree about the last one. -The server only names this template; the cluster fills in the rest. +Only rendered with a warm pool, because that is the only thing that reads it: a `Sandbox` carries its +pod inline and has no template reference, so the on-demand path takes the same shape from the +ConfigMap beside this file. One definition feeds both, so a pre-warmed computer and one created on +demand cannot drift into being different things. + +`extensions.agents.x-k8s.io`, not `agents.x-k8s.io`. The two extension kinds sit in their own API +group, which is easy to get wrong and fails as "no matches for kind" at install. */}} -apiVersion: agents.x-k8s.io/v1beta1 +apiVersion: extensions.agents.x-k8s.io/v1beta1 kind: SandboxTemplate metadata: name: {{ $ns }}-computer @@ -15,91 +19,5 @@ metadata: labels: {{ include "openbot.labels" . | indent 4 }} spec: - podTemplate: - metadata: - labels: -{{ include "openbot.componentSelectorLabels" (dict "root" . "component" "computer") | indent 8 }} - spec: - {{- with .Values.computers.runtimeClassName }} - runtimeClassName: {{ . }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: -{{ toYaml . | indent 8 }} - {{- end }} - terminationGracePeriodSeconds: 30 - {{- with .Values.computers.nodeSelector }} - nodeSelector: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.computers.tolerations }} - tolerations: -{{ toYaml . | indent 8 }} - {{- end }} - containers: - - name: computer - image: {{ include "openbot.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["/usr/local/bin/bun", "/app/agent-computer/src/index.ts"] - ports: - - name: http - containerPort: 4100 - env: - - name: PORT - value: "4100" - - name: WORKSPACE_DIR - value: /workspace - - name: PROFILES_DIR - value: /profiles - - name: COMPUTER_TOKEN - valueFrom: - secretKeyRef: - name: {{ default (include "openbot.secretName" .) .Values.computers.existingTokenSecret }} - key: computer-token - {{- with .Values.computers.extraEnv }} -{{ toYaml . | indent 12 }} - {{- end }} - volumeMounts: - - name: profiles - mountPath: /profiles - - name: workspace - mountPath: /workspace - {{- /* Readiness only. A browser slow under load is not a browser to restart. */}} - readinessProbe: - httpGet: - path: /health - port: http - periodSeconds: 10 - failureThreshold: 6 - {{- with .Values.computers.resources }} - resources: -{{ toYaml . | indent 12 }} - {{- end }} - {{- /* - The volumes that make a suspend worth doing. - - Suspending terminates the pod and keeps these, which is the difference between a computer that - comes back with its logins and one that comes back signed out of everything. - */}} - volumeClaimTemplates: - - metadata: - name: profiles - spec: - accessModes: ["ReadWriteOnce"] - {{- with .Values.computers.persistence.storageClass }} - storageClassName: {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .Values.computers.persistence.profilesSize }} - - metadata: - name: workspace - spec: - accessModes: ["ReadWriteOnce"] - {{- with .Values.computers.persistence.storageClass }} - storageClassName: {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .Values.computers.persistence.workspaceSize }} +{{ include "openbot.sandboxPodTemplate" . | fromJson | toYaml | indent 2 }} {{- end }} diff --git a/charts/openbot/templates/computer/warmpool.yaml b/charts/openbot/templates/computer/warmpool.yaml index f91ab0b1..1cac665b 100644 --- a/charts/openbot/templates/computer/warmpool.yaml +++ b/charts/openbot/templates/computer/warmpool.yaml @@ -8,7 +8,7 @@ from the old node first. That is a real wait, and this is the upstream mechanism of it rather than something to reinvent. Off by default, because a pool is browsers nobody is using yet, which is exactly the cost the suspend is here to remove. */}} -apiVersion: agents.x-k8s.io/v1beta1 +apiVersion: extensions.agents.x-k8s.io/v1beta1 kind: SandboxWarmPool metadata: name: {{ $ns }}-computer diff --git a/charts/openbot/templates/migrations/job.yaml b/charts/openbot/templates/migrations/job.yaml index d6d1783a..6d57a225 100644 --- a/charts/openbot/templates/migrations/job.yaml +++ b/charts/openbot/templates/migrations/job.yaml @@ -48,8 +48,16 @@ spec: {{ include "openbot.componentSelectorLabels" (dict "root" . "component" $component) | indent 8 }} spec: restartPolicy: Never - serviceAccountName: {{ include "openbot.serviceAccountName" . }} - automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- /* + NO SERVICE ACCOUNT, WHICH IS BOTH CORRECT AND NECESSARY. + + Correct because this Job talks to a database and never to the cluster, so a cluster + credential in it is one nothing needs. Necessary because as a pre-install hook it runs before + the chart's own resources exist, and naming the ServiceAccount the API pods use fails with + "serviceaccount not found" and a Job that can never schedule. The default account, with no + token mounted, is what a migration actually requires. + */}} + automountServiceAccountToken: false {{- with .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml . | indent 8 }} diff --git a/charts/openbot/templates/server/deployment.yaml b/charts/openbot/templates/server/deployment.yaml index abbe4b6c..5a044bca 100644 --- a/charts/openbot/templates/server/deployment.yaml +++ b/charts/openbot/templates/server/deployment.yaml @@ -32,12 +32,23 @@ spec: the old values, and the deployment looks upgraded while behaving exactly as it did. */}} checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if eq .Values.computers.mode "sandbox" }} + {{- /* + The shape of a computer, which the server reads once and keeps. + + A mounted ConfigMap does update in place, but the file is read on the first computer request + and the provider built from it is held for the life of the process. Without this the + template can change under a running server and every Sandbox it creates is still cut from + the old one, which is a rollout that looks like it worked and changes nothing. + */}} + checksum/computer-template: {{ include (print $.Template.BasePath "/computer/pod-template.yaml") . | sha256sum }} + {{- end }} {{- with .Values.server.podAnnotations }} {{ toYaml . | indent 8 }} {{- end }} spec: serviceAccountName: {{ include "openbot.serviceAccountName" . }} - automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + automountServiceAccountToken: {{ include "openbot.automountToken" . }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml . | indent 8 }} @@ -114,7 +125,20 @@ spec: periodSeconds: {{ .Values.server.livenessProbe.periodSeconds }} timeoutSeconds: {{ .Values.server.livenessProbe.timeoutSeconds }} failureThreshold: {{ .Values.server.livenessProbe.failureThreshold }} + {{- if eq .Values.computers.mode "sandbox" }} + {{- /* The shape of a Bot's computer, as a file rather than as a permission to read one. */}} + volumeMounts: + - name: sandbox-template + mountPath: /etc/openbot + readOnly: true + {{- end }} {{- with .Values.server.resources }} resources: {{ toYaml . | indent 12 }} {{- end }} + {{- if eq .Values.computers.mode "sandbox" }} + volumes: + - name: sandbox-template + configMap: + name: {{ include "openbot.componentName" (dict "root" . "component" "computer-template") }} + {{- end }} diff --git a/charts/openbot/templates/server/service.yaml b/charts/openbot/templates/server/service.yaml index b9bcda46..1d60306d 100644 --- a/charts/openbot/templates/server/service.yaml +++ b/charts/openbot/templates/server/service.yaml @@ -11,6 +11,10 @@ metadata: {{- end }} spec: type: {{ .Values.server.service.type }} + {{- with .Values.server.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml . | indent 4 }} + {{- end }} ports: - port: {{ .Values.server.service.port }} targetPort: http diff --git a/charts/openbot/templates/server/serviceaccount.yaml b/charts/openbot/templates/server/serviceaccount.yaml index a4c19681..83ddfa90 100644 --- a/charts/openbot/templates/server/serviceaccount.yaml +++ b/charts/openbot/templates/server/serviceaccount.yaml @@ -13,5 +13,5 @@ metadata: annotations: {{ toYaml . | indent 4 }} {{- end }} -automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ include "openbot.automountToken" . }} {{- end }} diff --git a/charts/openbot/templates/validation.yaml b/charts/openbot/templates/validation.yaml index c7ec4146..46fe4cac 100644 --- a/charts/openbot/templates/validation.yaml +++ b/charts/openbot/templates/validation.yaml @@ -52,6 +52,17 @@ This template renders nothing. {{- fail "config.auth.okta.issuer is required alongside the Okta client, such as https://example.okta.com/oauth2/default. It is what makes it a particular Okta rather than Okta in general." }} {{- end }} +{{- /* + Every visitor as one administrator, on an address the internet can reach. + + `singleUser` is a local trial mode, and a LoadBalancer with no source ranges is the open internet. + Together they are a deployment where anybody who finds the address is an administrator, which is + the exact failure the server refuses to start into when nobody has said they meant it. +*/}} +{{- if and .Values.config.singleUser (eq .Values.server.service.type "LoadBalancer") (not .Values.server.service.loadBalancerSourceRanges) }} +{{- fail "config.singleUser treats every visitor as an administrator, so it must not be put behind a LoadBalancer that anybody can reach. Set server.service.loadBalancerSourceRanges, or configure an identity provider." }} +{{- end }} + {{- if and .Values.config.singleUser .Values.config.publicUrl }} {{- fail "config.singleUser serves every visitor as one administrator, so it must not be combined with a public URL. Configure an identity provider and set config.initialAdminEmails instead." }} {{- end }} @@ -83,6 +94,24 @@ This template renders nothing. {{- fail "computers.mode is external but computers.url is empty, so no Bot would have a computer. Set the address, or use mode: shared to have this chart run one." }} {{- end }} +{{- /* + Asking for per-Bot computers on a cluster that cannot make them. + + `computers.mode: sandbox` creates `Sandbox` objects, which only exist once the agent-sandbox + controller is installed. Without it the install succeeds, every pod is healthy, and the deployment + looks finished right up until the first Bot asks for a browser and gets a 404 from the API server. + That is the worst time to learn it, so it is a refusal here instead, with the command to run. + + Read from the cluster, so it is a real check rather than a value somebody has to remember to set. + `helm template` with no cluster has no way to know, which is what `--api-versions` is for and what + the chart's own render tests pass. +*/}} +{{- if and (eq .Values.computers.mode "sandbox") .Values.computers.sandbox.requireController }} +{{- if not (.Capabilities.APIVersions.Has "agents.x-k8s.io/v1beta1/Sandbox") }} +{{- fail "computers.mode is sandbox, which gives every Bot its own computer, but this cluster has no Sandbox CRD. Install the controller first:\n\n kubectl apply --server-side -f https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v0.5.6/sandbox-with-extensions.yaml\n\nOr use computers.mode: shared, which needs nothing installed. Rendering without a cluster? Pass --api-versions agents.x-k8s.io/v1beta1/Sandbox, or set computers.sandbox.requireController=false." }} +{{- end }} +{{- end }} + {{- if not (has .Values.computers.mode (list "shared" "sandbox" "external")) }} {{- fail "computers.mode must be one of: shared, sandbox, external." }} {{- end }} diff --git a/charts/openbot/values.yaml b/charts/openbot/values.yaml index e4137214..81585c51 100644 --- a/charts/openbot/values.yaml +++ b/charts/openbot/values.yaml @@ -63,6 +63,11 @@ server: type: ClusterIP port: 3001 annotations: {} + # WHO MAY REACH IT, when the type is LoadBalancer. Empty means everybody, which is what a cloud + # load balancer does by default and almost never what somebody wants for an internal tool. Narrow + # it to the addresses your people come from. Not every cloud honours this on every load balancer + # type, so treat it as one layer rather than the only one. + loadBalancerSourceRanges: [] # Probes hit the API's own health route. livenessProbe: @@ -197,6 +202,10 @@ computers: # `mode: sandbox` only. Where the per-Bot computers are created and what may create them. sandbox: + # Refuse to install when the cluster has no Sandbox CRD, rather than succeeding and failing at + # the first browser action. Turn it off only where the CRD arrives after this chart does, such as + # a GitOps run that applies both together. + requireController: true namespace: "" # Pre-warmed sandboxes, so a Bot's first action after lunch does not wait for Chromium to boot. warmPool: @@ -211,11 +220,29 @@ computers: database: # Used when `postgresql.enabled` is false. A URL, or a secret holding one. + # + # PUT `?sslmode=require` ON IT. Every managed database refuses an unencrypted connection: RDS has + # `rds.force_ssl` on by default, and Cloud SQL and Azure Database do the same. Without it the + # migration fails with `no pg_hba.conf entry for host ... no encryption`, which names the host and + # the user and not the actual problem. + # + # postgres://user:password@host:5432/openbot?sslmode=require + # + # Keep it in a Secret rather than here: `url` is readable by anybody who can read the release. url: "" existingSecret: "" existingSecretKey: "database-url" # The bundled database. Off by default in favour of a managed one. +# The bundled database. +# +# NOT PRODUCTION-GRADE, AND OFF FOR THAT REASON. A database on a pod is a database that goes away +# when the pod does: a rollout, a node drain or an evicted pod is a restart, and while the volume +# survives, nothing about this shape gives you backups, failover, point-in-time recovery or a +# connection that outlives the release. It exists so somebody can try OpenBot in one command. +# +# A REAL DEPLOYMENT POINTS `database` ABOVE AT A MANAGED DATABASE: RDS, Cloud SQL, Azure Database, or +# your own server. That is the same line the Intelligence chart draws, and for the same reason. postgresql: enabled: false auth: diff --git a/server/src/computer/provider.ts b/server/src/computer/provider.ts index 4a24b8fe..a6258ffc 100644 --- a/server/src/computer/provider.ts +++ b/server/src/computer/provider.ts @@ -1,5 +1,9 @@ import type { ComputerConfig } from "../config"; -import { createSandboxComputerProvider, inClusterConfig } from "./sandbox"; +import { + createSandboxComputerProvider, + inClusterConfig, + readSandboxTemplate, +} from "./sandbox"; import type { ComputerStatus } from "./schema"; import { createDockerSupervisorProvider, @@ -257,11 +261,14 @@ function createLazySandboxProvider( const provider = async (): Promise => { built ??= (async () => { - const cluster = await inClusterConfig(); + const [cluster, template] = await Promise.all([ + inClusterConfig(), + readSandboxTemplate(config.templateFile), + ]); return createSandboxComputerProvider({ namespace: config.namespace, idleAfterMs: config.idleAfterMs, - template: sandboxPodTemplate(config), + template, apiServer: cluster.apiServer, token: cluster.token, ca: cluster.ca, @@ -281,16 +288,3 @@ function createLazySandboxProvider( sessionOf: async (botId) => (await provider()).sessionOf?.(botId), }; } - -/** - * The pod every Bot's computer is cut from. - * - * Read from the `SandboxTemplate` the chart installs rather than written here: what image a computer - * runs, what volumes it keeps and what runtime class it uses are deployment decisions, and the three - * clouds disagree about the last one. The server only needs to know that a template exists. - */ -function sandboxPodTemplate( - config: Extract, -): Record { - return { sandboxTemplateRef: { name: `${config.namespace}-computer` } }; -} diff --git a/server/src/computer/sandbox.ts b/server/src/computer/sandbox.ts index 4f851759..3bce0a48 100644 --- a/server/src/computer/sandbox.ts +++ b/server/src/computer/sandbox.ts @@ -44,7 +44,14 @@ type Sandbox = { export type SandboxProviderOptions = { /** Where the Bots' computers live. The provider is scoped to exactly this namespace. */ namespace: string; - /** The pod template every computer is cut from, as YAML-derived JSON from the chart. */ + /** + * The pod and volumes every computer is cut from. + * + * `Sandbox` carries its pod inline and has no template reference, so this has to come from + * somewhere. It comes from a file the chart mounts, which means the server needs no permission to + * read ConfigMaps and the shape of a computer stays a deployment decision rather than a constant + * in this file. + */ template: Record; /** How long a computer may go untouched before the culler suspends it. */ idleAfterMs: number; @@ -63,6 +70,33 @@ export type SandboxProviderOptions = { * this provider when a deployment configured it, and a clear message about a missing token beats a * connection refused to an address nobody set. */ +/** + * Read the pod template the chart mounted. + * + * A missing file is a deployment that asked for per-Bot computers without saying what one looks + * like, which is worth failing on by name rather than creating a Sandbox the API server rejects for + * a missing required field. + */ +export async function readSandboxTemplate( + path: string, +): Promise> { + let raw: string; + try { + raw = await readFile(path, "utf8"); + } catch { + throw new SandboxError( + `COMPUTER_SANDBOX_TEMPLATE_FILE points at ${path}, which cannot be read. That file is what a Bot's computer is cut from; the chart mounts it when computers.mode is sandbox.`, + ); + } + const parsed = JSON.parse(raw) as Record; + if (!parsed.podTemplate) { + throw new SandboxError( + `${path} has no podTemplate, so there is nothing to make a computer from.`, + ); + } + return parsed; +} + export async function inClusterConfig(): Promise<{ apiServer: string; token: string; @@ -134,13 +168,22 @@ export function createSandboxComputerProvider( const { contentType, ...rest } = init; const response = await doFetch(`${base()}${path}`, { ...rest, + /* + * The cluster's own CA, which is the only thing that signs the API server's certificate. + * + * It is not in any public trust store, so without this every call fails with "unable to verify + * the first certificate" and a Bot simply never gets a computer. The alternative some reach for + * is to stop verifying, which would leave the token in every one of these requests open to + * anything that can answer on that address. + */ + ...(options.ca ? { tls: { ca: options.ca } } : {}), headers: { ...(options.token ? { authorization: `Bearer ${options.token}` } : {}), ...(contentType ? { "content-type": contentType } : {}), accept: "application/json", ...(rest.headers ?? {}), }, - }); + } as RequestInit); if (response.status === 404) return undefined; if (!response.ok) { const body = await response.text().catch(() => ""); @@ -170,7 +213,12 @@ export function createSandboxComputerProvider( // rather than trying to reverse the slug. annotations: { "openbot.dev/bot-id": botId }, }, - spec: { operatingMode: "Running", ...options.template }, + spec: { + operatingMode: "Running", + // `podTemplate` is required and `volumeClaimTemplates` is what makes a suspend worth doing, + // and both arrive together from the mounted template. + ...options.template, + }, }; } diff --git a/server/src/config.ts b/server/src/config.ts index ad4422d4..3dced0e6 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -49,6 +49,8 @@ export type SandboxComputerConfig = { provider: "sandbox"; namespace: string; idleAfterMs: number; + /** Where the chart mounted the shape of a computer. */ + templateFile: string; token?: string; allowPrivateHosts: boolean; policy?: ActionPolicy; @@ -637,6 +639,9 @@ function computerConfig(environment: Environment): ComputerConfig | undefined { idleAfterMs: durationMs( optional(environment, "COMPUTER_SANDBOX_IDLE_AFTER") ?? "30m", ), + templateFile: + optional(environment, "COMPUTER_SANDBOX_TEMPLATE_FILE") ?? + "/etc/openbot/sandbox-template.json", allowPrivateHosts, ...(computerToken ? { token: computerToken } : {}), ...(policy ? { policy } : {}), From 8da5e92b24d88db9ceff79c2eddbdd424ed7653b Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 24 Aug 2026 17:48:16 -0700 Subject: [PATCH 4/9] Tell one run of a computer from the next across a suspend A resumed browser counts snapshot generations from one again, so a ref the model still holds from before the suspend matches a row nothing has overwritten, and the boundary decides about an element on a page that no longer exists. The first answer used the node and the pod address. Resuming a real computer on EKS disproved it: a suspended sandbox is very often rescheduled onto the same node and handed the same address back, and both were identical across the cycle, so the check would have said same run for the exact case it exists to catch. The Ready condition's transition time moves whenever a computer starts serving again, needs no permission beyond the sandbox already read, and is precisely the question. Driven on EKS: a ref taken before a suspend is refused after the resume, naming why, and a fresh ref from a new snapshot clicks through. --- CHANGELOG.md | 9 ++ server/src/computer/sandbox.ts | 38 +++++--- server/tests/computer-sandbox.test.ts | 125 ++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 10 deletions(-) create mode 100644 server/tests/computer-sandbox.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 91e2eba6..0d1c7161 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,15 @@ hand-offs between Bots are the other two, which is why it is written once rather slightly differently. A CronJob runs the sweep, because a timer in the API fires in every replica and suspending a browser somebody just started using is not something to do five times. +**Which run of a computer this is, across a suspend.** A resumed browser counts snapshot +generations from one again, so a ref the model still holds from before the suspend would match a row +nothing has overwritten and the boundary would decide about an element on a page that no longer +exists. The first answer here used the node and the pod address, and resuming a real computer +disproved it: a suspended sandbox is very often rescheduled onto the same node and handed the same +address back, so both were identical across a suspend and resume and the check would have said "same +run" for the exact case it exists to catch. It reads the `Ready` condition's transition time instead, +which moves every time a computer starts serving again. + **Which run of a computer this is, on more than one replica.** `sessionOf` answered from a map in the process that started the computer, which is right until there are two: the replica that took a snapshot is usually not the one handling the click, and the second had nothing to answer with. An diff --git a/server/src/computer/sandbox.ts b/server/src/computer/sandbox.ts index 3bce0a48..0c0e170d 100644 --- a/server/src/computer/sandbox.ts +++ b/server/src/computer/sandbox.ts @@ -35,7 +35,13 @@ type Sandbox = { spec?: { operatingMode?: "Running" | "Suspended" }; status?: { serviceFQDN?: string; - conditions?: { type?: string; status?: string; reason?: string }[]; + conditions?: { + type?: string; + status?: string; + reason?: string; + /** When this condition last changed, which is how one run of a computer is told from the next. */ + lastTransitionTime?: string; + }[]; podIPs?: string[]; nodeName?: string; }; @@ -347,20 +353,32 @@ export function createSandboxComputerProvider( /* * Which run of this computer this is, and it has to change across a suspend and resume. * - * A snapshot's generation only orders snapshots within one run of a browser: a resumed - * computer counts from one again, so a ref the model still holds from before the suspend would - * match a row nothing has overwritten and the boundary would decide about an element on a page - * that no longer exists. The pod's address changes when it is rescheduled, and the node it - * landed on with it, so the two together identify the run without asking the browser anything. + * A snapshot's generation only orders snapshots within one run of a browser: a resumed computer + * counts from one again, so a ref the model still holds from before the suspend would match a + * row nothing has overwritten, and the boundary would decide about an element on a page that no + * longer exists. + * + * NOT THE NODE AND NOT THE POD IP, which is what this used and what testing a real resume + * disproved. A suspended sandbox is very often rescheduled onto the same node and handed the + * same address back, because nothing else has taken it: measured on EKS, both were byte for + * byte identical across a suspend and resume, so the check would have said "same run" for the + * exact case it exists to catch. + * + * The `Ready` condition's transition time does move, because suspending drives Ready to False + * and resuming drives it back to True. It is the moment this run of the browser started + * serving, which is precisely the question, and it needs no permission beyond the sandbox this + * already reads. The pod's own UID would be exact too, and would cost a second read and the + * right to list pods. * * Reading, never ensuring: this must not be the thing that wakes a computer up. */ const sandbox = await read(botId); if (!sandbox || isSuspended(sandbox)) return undefined; - const ip = sandbox.status?.podIPs?.[0]; - const node = sandbox.status?.nodeName; - if (!ip && !node) return undefined; - return [node, ip].filter(Boolean).join("/"); + const ready = sandbox.status?.conditions?.find( + (condition) => condition.type === "Ready", + ); + if (ready?.status !== "True") return undefined; + return ready.lastTransitionTime; }, }; } diff --git a/server/tests/computer-sandbox.test.ts b/server/tests/computer-sandbox.test.ts new file mode 100644 index 00000000..e26f504d --- /dev/null +++ b/server/tests/computer-sandbox.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, test } from "bun:test"; +import { + createSandboxComputerProvider, + sandboxNameFor, +} from "../src/computer/sandbox"; + +/** + * A computer each, as a Sandbox, and the two questions that decide whether it is safe. + * + * Which run of a computer this is, because a resumed browser counts generations from one again and a + * ref from before the suspend must not resolve against the page after it. And whether asking a + * question can wake a computer, because one that wakes on being asked about never suspends and the + * bill never falls. + */ +function providerWith(sandbox: unknown, seen: string[] = []) { + return createSandboxComputerProvider({ + namespace: "openbot", + template: { podTemplate: { spec: { containers: [] } } }, + idleAfterMs: 60_000, + apiServer: "https://kubernetes.default", + token: "t", + fetchImpl: (async (url: string | URL | Request, init?: RequestInit) => { + seen.push(`${init?.method ?? "GET"} ${new URL(String(url)).pathname}`); + return Response.json(sandbox); + }) as unknown as typeof fetch, + }); +} + +const running = (readyAt: string, node: string, ip: string) => ({ + metadata: { name: "bot-knowledge-abc" }, + spec: { operatingMode: "Running" }, + status: { + serviceFQDN: "bot-knowledge-abc.openbot.svc.cluster.local", + nodeName: node, + podIPs: [ip], + conditions: [ + { type: "Suspended", status: "False" }, + { type: "Ready", status: "True", lastTransitionTime: readyAt }, + ], + }, +}); + +describe("telling one run of a Bot's computer from the next", () => { + /* + * THE CASE A REAL RESUME DISPROVED THE OLD ANSWER WITH. + * + * A suspended sandbox is very often rescheduled onto the same node and handed the same address + * back, because nothing else has taken it. Measured on EKS: both identical across a suspend and + * resume. Anything built from them says "same run" for the exact case the check exists to catch. + */ + test("changes across a resume even when the node and address do not", async () => { + const before = await providerWith( + running("2026-08-24T23:14:04Z", "node-a", "192.168.49.27"), + ).sessionOf?.("knowledge"); + const after = await providerWith( + running("2026-08-25T00:40:04Z", "node-a", "192.168.49.27"), + ).sessionOf?.("knowledge"); + + expect(before).toBeDefined(); + expect(after).toBeDefined(); + expect(after).not.toBe(before); + }); + + test("is the same while one run keeps serving", async () => { + const sandbox = running("2026-08-25T00:40:04Z", "node-a", "192.168.49.27"); + expect(await providerWith(sandbox).sessionOf?.("knowledge")).toBe( + await providerWith(sandbox).sessionOf?.("knowledge"), + ); + }); + + test("a suspended computer has no run to name", async () => { + // Unknown rather than mismatched: there is no page behind a suspended computer to resolve against. + const suspended = { + metadata: { name: "bot-knowledge-abc" }, + spec: { operatingMode: "Suspended" }, + status: { conditions: [{ type: "Suspended", status: "True" }] }, + }; + expect( + await providerWith(suspended).sessionOf?.("knowledge"), + ).toBeUndefined(); + }); + + test("asking which run it is never starts a computer", async () => { + /* + * The invisible way to lose scale-to-zero: everything works, nothing ever suspends, and only the + * bill says otherwise. Reading is a GET; anything that creates or patches would wake it. + */ + const seen: string[] = []; + await providerWith( + running("2026-08-25T00:40:04Z", "n", "1.2.3.4"), + seen, + ).sessionOf?.("knowledge"); + expect(seen.every((call) => call.startsWith("GET "))).toBe(true); + }); + + test("status reads a suspended computer as down and fine, without touching it", async () => { + const seen: string[] = []; + const suspended = { + metadata: { name: "bot-knowledge-abc" }, + spec: { operatingMode: "Suspended" }, + status: { conditions: [{ type: "Suspended", status: "True" }] }, + }; + const status = await providerWith(suspended, seen).status("knowledge"); + + expect(status.state).toBe("absent"); + expect(seen.every((call) => call.startsWith("GET "))).toBe(true); + }); +}); + +describe("naming a Bot's computer in a cluster", () => { + test("a bot id that is not a legal name still gets one, and a unique one", () => { + // Bot ids are ours and hold anything a person typed; a resource name may not. Two ids that differ + // only in punctuation must not land on one computer, which would be one Bot reading another's + // logins. + const a = sandboxNameFor("Sales Bot"); + const b = sandboxNameFor("sales-bot"); + expect(a).toMatch(/^[a-z0-9-]+$/); + expect(b).toMatch(/^[a-z0-9-]+$/); + expect(a).not.toBe(b); + }); + + test("the same id always names the same computer", () => { + expect(sandboxNameFor("knowledge")).toBe(sandboxNameFor("knowledge")); + }); +}); From 4c6498ee3c9777e2d9af97e0f6c0e29ca27b60ab Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 24 Aug 2026 17:52:16 -0700 Subject: [PATCH 5/9] Let the policy reach the computers, and refuse one that fences off the database The NetworkPolicy allowed DNS and the bundled database. Nothing let the API reach a Bot's computer, which it does for every browser action, and nothing let it reach a managed database, whose address this chart cannot know. On a cluster that enforces policy both are outages that read as something else: the API looks broken rather than fenced. The computers and the API server are allowed now, and turning the policy on with an external database and no rule for it is refused with the shape of the rule to add. None of this showed up by installing it, because EKS runs its CNI with --enable-network-policy=false and the policy is inert there. That is worth knowing on its own, so it is written down: a policy that installs, looks right, and does nothing is worse than one that is off. Also driven on EKS: reset takes the volumes with it and the Bot gets a clean profile afterwards, and the HPA reads real metrics. --- CHANGELOG.md | 7 ++++++ charts/openbot/README.md | 19 ++++++++++++++++ charts/openbot/templates/networkpolicy.yaml | 24 +++++++++++++++++++++ charts/openbot/templates/validation.yaml | 15 +++++++++++++ 4 files changed, 65 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d1c7161..86c5b1cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,13 @@ isolated, stateful, singleton pod with a stable identity and persistent storage. field, and it keeps the volumes, so a computer comes back with its logins rather than signed out of everything. `shared` stays the default and needs nothing installed in the cluster. +**The NetworkPolicy would have fenced the API off from its own work.** Its egress named DNS and the +bundled database and nothing else, so on a cluster that enforces policy the API could not have +reached a Bot's computer or, with a managed database, the database. Both are allowed now, and turning +the policy on with an external database and no rule for it is refused rather than shipped. Worth +knowing either way: EKS runs its CNI with `--enable-network-policy=false`, so a policy there installs, +looks right, and does nothing at all. + **A cluster with no controller is refused at install.** `computers.mode: sandbox` needs the agent-sandbox CRD, and without it the install succeeds, every pod is healthy, and the deployment looks finished until the first Bot asks for a browser. The chart reads the cluster and refuses, diff --git a/charts/openbot/README.md b/charts/openbot/README.md index 9aaac36e..e2e9f5c8 100644 --- a/charts/openbot/README.md +++ b/charts/openbot/README.md @@ -165,6 +165,25 @@ PostgreSQL with `select ... for update skip locked`, so whichever pod runs the s nobody else holds, and one that dies mid-suspend hands its work back when the lease expires. The decision is re-checked at the moment of acting, because somebody may have come back in between. +## NetworkPolicy, and whether your cluster enforces one + +Off by default, because a NetworkPolicy on a cluster whose CNI does not enforce one is a resource +that silently does nothing, and on a cluster that does enforce one a wrong rule is an outage. + +**On EKS it does nothing unless you turn it on.** The VPC CNI ships with +`--enable-network-policy=false`, so the policy installs, looks right, and is never applied. Check +before trusting it: + +```sh +kubectl -n kube-system get ds aws-node -o yaml | grep enable-network-policy +``` + +The egress rules allow DNS, a Bot's computer on 4100, the API server when computers are Sandboxes, +and the bundled database. **A managed database is an address this chart cannot know**, so turning +the policy on with an external database and no `networkPolicy.extraEgress` is refused: on an +enforcing cluster it would fence the API off from its own database, which reads as the database +being down. + ## Upgrades Migrations run as a `pre-install,pre-upgrade` Job, so no replica ever serves in front of a schema it diff --git a/charts/openbot/templates/networkpolicy.yaml b/charts/openbot/templates/networkpolicy.yaml index 1f21b02f..3da0bdf9 100644 --- a/charts/openbot/templates/networkpolicy.yaml +++ b/charts/openbot/templates/networkpolicy.yaml @@ -46,6 +46,30 @@ spec: - port: 5432 protocol: TCP {{- end }} + {{- if ne .Values.computers.mode "external" }} + {{- /* + A Bot's computer, which the API has to reach for every browser action. + + Easy to forget, because nothing about the API looks like it talks to another pod: the browser is + behind a Service and reads like an outside address. On a cluster that enforces policy, leaving + this out means every Bot action fails and the API looks broken rather than fenced. + */}} + - to: + - podSelector: + matchLabels: + app.kubernetes.io/component: computer + ports: + - port: 4100 + protocol: TCP + {{- end }} + {{- if eq .Values.computers.mode "sandbox" }} + {{- /* The API server, which is where a per-Bot computer is asked for. */}} + - ports: + - port: 443 + protocol: TCP + - port: 6443 + protocol: TCP + {{- end }} {{- with .Values.networkPolicy.extraEgress }} {{ toYaml . | indent 4 }} {{- end }} diff --git a/charts/openbot/templates/validation.yaml b/charts/openbot/templates/validation.yaml index 46fe4cac..9b93933a 100644 --- a/charts/openbot/templates/validation.yaml +++ b/charts/openbot/templates/validation.yaml @@ -116,6 +116,21 @@ This template renders nothing. {{- fail "computers.mode must be one of: shared, sandbox, external." }} {{- end }} +{{- /* + A policy that would fence the deployment off from its own database. + + The egress rules name DNS, the computers and, when it is bundled, the database. A managed database + is an address this chart cannot know, so on a cluster that enforces policy the API would resolve it + and then be unable to reach it, which reads as the database being down rather than as a rule. + + Worth knowing either way: a NetworkPolicy on a cluster whose CNI does not enforce one is a resource + that silently does nothing. EKS needs the VPC CNI started with `--enable-network-policy=true`, and + it is off by default, so this can look like it is working when it is not doing anything at all. +*/}} +{{- if and .Values.networkPolicy.enabled (not .Values.postgresql.enabled) (not .Values.networkPolicy.extraEgress) }} +{{- fail "networkPolicy.enabled with an external database, but nothing lets the API reach it. Add the database to networkPolicy.extraEgress, for example a `to: [{ipBlock: {cidr: 10.0.0.0/16}}]` with port 5432." }} +{{- end }} + {{- if and .Values.ingress.enabled .Values.httpRoute.enabled }} {{- fail "ingress.enabled and httpRoute.enabled are both set. They are two ways to do the same thing; pick the one your cluster runs." }} {{- end }} From 5bf444c4cfa871c90e0006bac4ce74b9b07d2242 Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 24 Aug 2026 18:18:06 -0700 Subject: [PATCH 6/9] Keep the browsing that produced an answer Every turn in which a Bot used a tool vanished from the transcript on reload. The sentence the Bot wrote stayed, the browsing that produced it did not, the inline screen went with it, and the footer said some messages could not be read. The history store writes a tool call as {id, name, args}; AG-UI describes {id, type: function, function: {name, arguments}}. The reader validated against the second and treated the first as damage from an interrupted run. It is not damage, it is how every tool call is stored, so a guard written against one bad turn was deleting all the real ones. Found by driving a real conversation on the EKS deployment rather than by reading: two browsing turns, both counted unreadable, both well formed in the store's own dialect. Both spellings now read as the same thing. A mixed or unrecognised array is still refused rather than half-translated, because a reader that rewrites what it does not recognise is worse than one that refuses it. --- CHANGELOG.md | 18 +++ app/src/lib/copilot/thread-messages.ts | 61 ++++++- app/tests/thread-messages.test.ts | 212 ++++++++++--------------- 3 files changed, 161 insertions(+), 130 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86c5b1cb..0dc22072 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,24 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### A conversation keeps the browsing that produced its answers + +Every turn in which a Bot used a tool was disappearing from the transcript on reload. The sentence +the Bot wrote stayed; the browsing that produced it did not, the inline screen went with it, and the +footer said some messages could not be read. + +The history store writes a tool call as `{id, name, args}`. AG-UI describes +`{id, type: "function", function: {name, arguments}}`. The reader validated against the second, +treated the first as damage from an interrupted run, and dropped it. It is not damage: it is how +every tool call is stored, so what looked like a guard against one bad turn was deleting all of the +real ones. Observed on a live thread where every browsing turn was counted unreadable and every one +of them was well formed in the store's own dialect. + +The two spellings are now read as the same thing. The check stays for turns that really are +malformed, and a mixed or unrecognised array is still refused rather than half-translated, because a +reader that rewrites what it does not recognise is worse than one that refuses it. + + ### Run this on Kubernetes A Helm chart under `charts/openbot`, Bots and all, and the fixes that installing it for real turned diff --git a/app/src/lib/copilot/thread-messages.ts b/app/src/lib/copilot/thread-messages.ts index 887ecec3..4cbb1c87 100644 --- a/app/src/lib/copilot/thread-messages.ts +++ b/app/src/lib/copilot/thread-messages.ts @@ -11,14 +11,25 @@ import { tryClient } from "@/lib/client"; * * WHAT ARRIVES HERE IS NOT TRUSTED. This used to end `stored as Message[]`, which is a cast rather * than a check: whatever the history store held was handed to `setMessages` and then to every - * projection that reads a transcript. A turn shaped differently — a tool call persisted as - * `{id, name, args}` instead of AG-UI's `{id, type: "function", function: {…}}`, which interrupted - * runs have produced — reached a renderer that dereferenced `toolCall.function.arguments` and took - * the whole conversation down with it. One bad turn made a thread unreadable. + * projection that reads a transcript. A turn shaped differently reached a renderer that dereferenced + * `toolCall.function.arguments` and took the whole conversation down with it. One bad turn made a + * thread unreadable. * * So each turn is parsed against the schema AG-UI ships, and one that does not parse is left out. * Checked here rather than in a projection because there are several projections and one history: * fixing it in the reader that is closest to the wire is what makes every consumer safe at once. + * + * BUT `{id, name, args}` IS NOT A CORRUPTION, AND TREATING IT AS ONE DELETED REAL WORK. That shape + * was read as damage from an interrupted run and dropped. It is how the runtime persists every tool + * call it stores, so dropping it meant every turn in which a Bot used a tool vanished on reload: the + * transcript kept the sentence the Bot wrote and lost the browsing that produced it, the inline + * screen went with it, and the footer said some messages could not be read. Observed against a live + * thread, where every browsing turn was counted unreadable and every one of them was well formed in + * the store's own dialect. + * + * So it is translated rather than refused. The check stays for turns that really are malformed; a + * reader is entitled to insist on one shape, but not to throw away the history because the writer + * spells it another way. */ /** @@ -52,8 +63,9 @@ export function readableTurns(stored: readonly unknown[]): StoredThread { let unreadable = 0; for (const turn of stored) { - if (MessageSchema.safeParse(turn).success) { - messages.push(turn as Message); + const candidate = withNormalisedToolCalls(turn); + if (MessageSchema.safeParse(candidate).success) { + messages.push(candidate as Message); } else { unreadable += 1; } @@ -62,6 +74,43 @@ export function readableTurns(stored: readonly unknown[]): StoredThread { return { messages, unreadable }; } +/** A tool call as the history store writes one. */ +type StoredToolCall = { id?: unknown; name?: unknown; args?: unknown }; + +/** + * The store's dialect for a tool call, in the shape AG-UI describes. + * + * `{id, name, args}` becomes `{id, type: "function", function: {name, arguments}}`. Only the array is + * rebuilt and only when every entry is in that dialect: a turn already in AG-UI's shape is returned + * untouched, and a mixed or unrecognised array is left exactly as it came so the parse below still + * refuses it rather than this quietly inventing something. + * + * The rest of the message is spread through unchanged, for the same reason `parsed.data` is not used + * anywhere here: a reader that rewrites what it does not recognise is worse than one that refuses it. + */ +function withNormalisedToolCalls(turn: unknown): unknown { + if (typeof turn !== "object" || turn === null) return turn; + const calls = (turn as { toolCalls?: unknown }).toolCalls; + if (!Array.isArray(calls) || calls.length === 0) return turn; + + const isStoredDialect = (call: unknown): call is StoredToolCall => + typeof call === "object" && + call !== null && + "name" in call && + "args" in call && + !("function" in call); + if (!calls.every(isStoredDialect)) return turn; + + return { + ...(turn as Record), + toolCalls: calls.map((call) => ({ + id: call.id, + type: "function", + function: { name: call.name, arguments: call.args }, + })), + }; +} + export async function readThreadMessages( threadId: string, agentId: string, diff --git a/app/tests/thread-messages.test.ts b/app/tests/thread-messages.test.ts index faa8f831..f1d4298c 100644 --- a/app/tests/thread-messages.test.ts +++ b/app/tests/thread-messages.test.ts @@ -1,159 +1,123 @@ import { describe, expect, test } from "bun:test"; -import { readableTurns } from "@/lib/copilot/thread-messages"; +import { readableTurns } from "../src/lib/copilot/thread-messages"; /** - * What comes back out of the history store, and what is refused at the door. + * Reading back a conversation that used a tool. * - * The old reader cast whatever it was given to `Message[]`, so a turn the store held in some other - * shape reached every projection that draws a transcript — and one of them dereferenced - * `toolCall.function.arguments`, which took the conversation down rather than the turn. - * - * These are the shapes that have actually been seen, not invented ones: the tool call persisted as - * `{id, name, args}` comes from #199, which found seventeen of them in one thread after interrupted - * runs, and the content shapes come from the tests on #43. + * The shapes below are copied from a live thread rather than invented. The store writes a tool call + * as `{id, name, args}`; AG-UI describes `{id, type: "function", function: {name, arguments}}`. A + * reader that insists on the second and refuses the first throws away every turn in which a Bot did + * anything, which is the half of the conversation worth keeping. */ - -const userTurn = { id: "m1", role: "user", content: "What did I miss?" }; - -const assistantTurn = { - id: "m2", - role: "assistant", - content: "Here is the summary.", +const userTurn = { + id: "6953d56c", + role: "user", + content: "open hackernews.com and tell me the top 3 stories", }; -/** As AG-UI defines a tool call: a literal `function` type, with the call nested under it. */ -const wellFormedToolCall = { - id: "m3", +/** As the history store writes it. */ +const storedToolCall = { + id: "0fe7b049", role: "assistant", toolCalls: [ { - id: "call_1", - type: "function", - function: { name: "search_files", arguments: "{}" }, + id: "call_maB4q3", + name: "computer_navigate", + args: '{"url":"https://news.ycombinator.com"}', }, ], }; -describe("reading back a stored thread", () => { - test("an ordinary conversation comes back whole, with nothing counted", () => { - const { messages, unreadable } = readableTurns([userTurn, assistantTurn]); - expect(messages).toHaveLength(2); - expect(unreadable).toBe(0); - }); +const toolResult = { + id: "aa5e9452", + role: "tool", + toolCallId: "call_maB4q3", + content: '{"ok":true,"title":"Hacker News"}', +}; - test("a well-formed tool call survives", () => { - // The shape the transcript knows how to draw. If validation rejected this, the fix would have - // traded a crash for an empty conversation. - const { messages, unreadable } = readableTurns([wellFormedToolCall]); - expect(messages).toHaveLength(1); - expect(unreadable).toBe(0); - }); - - test("a tool call stored the LangChain way is dropped and counted", () => { - /* - * The turn from #199: `{id, name, args}` rather than `{id, type: "function", function: {…}}`. - * This is the one that crashed a renderer reading `toolCall.function.arguments`, so the whole - * turn has to not arrive — and the count is what stops it vanishing quietly. - */ - const langChainShaped = { - id: "m4", - role: "assistant", - toolCalls: [{ id: "call_2", name: "search_files", args: {} }], - }; +const answer = { id: "5c1f", role: "assistant", content: "Top 3 stories…" }; +describe("restoring a conversation that used a tool", () => { + test("a browsing turn survives the read", () => { const { messages, unreadable } = readableTurns([ userTurn, - langChainShaped, - assistantTurn, + storedToolCall, + toolResult, + answer, ]); - expect(messages.map((message) => message.id)).toEqual(["m1", "m2"]); - expect(unreadable).toBe(1); - }); - - test("multimodal content is not mistaken for a malformed turn", () => { - // AG-UI allows content as typed parts as well as a string, so a turn carrying an image is - // ordinary. Rejecting it would lose real messages in the name of safety. - const withParts = { - id: "m5", - role: "user", - content: [{ type: "text", text: "What is in this?" }], - }; - const { messages, unreadable } = readableTurns([withParts]); - expect(messages).toHaveLength(1); + // Every one of them, and the tool call above all: without it the transcript keeps the sentence + // the Bot wrote and loses the browsing that produced it. + expect(messages).toHaveLength(4); expect(unreadable).toBe(0); }); - test("content that is not content is dropped rather than drawn as empty", () => { - /* - * From the #43 cases. These used to reach a projection and resolve to an empty message, so the - * transcript showed a turn that said nothing and read as though somebody had sent a blank line. - * Refused here instead, and reported. - */ - const shapes = [ - { id: "a", role: "user" }, - { id: "b", role: "user", content: null }, - { id: "c", role: "user", content: 42 }, - { id: "d", role: "user", content: [null, "text", 7] }, - ]; - - const { messages, unreadable } = readableTurns(shapes); - expect(messages).toEqual([]); - expect(unreadable).toBe(4); - }); - - test("a turn that is not an object at all is dropped", () => { - const { messages, unreadable } = readableTurns([ - null, - "a string", - 7, - userTurn, - ]); - expect(messages.map((message) => message.id)).toEqual(["m1"]); - expect(unreadable).toBe(3); - }); - - test("a turn with no recognised role is dropped", () => { - // The schema is a union on `role`, so an unknown one matches no member. - const { messages, unreadable } = readableTurns([ - { id: "x", role: "narrator", content: "once upon a time" }, - ]); - expect(messages).toEqual([]); - expect(unreadable).toBe(1); + test("the tool call comes back in the shape every renderer reads", () => { + const { messages } = readableTurns([storedToolCall]); + expect(messages[0]).toMatchObject({ + role: "assistant", + toolCalls: [ + { + id: "call_maB4q3", + type: "function", + function: { + name: "computer_navigate", + arguments: '{"url":"https://news.ycombinator.com"}', + }, + }, + ], + }); }); - test("order is the stored order, so a dropped turn does not reshuffle the rest", () => { - const { messages } = readableTurns([ - assistantTurn, - { id: "bad", role: "assistant", toolCalls: [{ id: "c", name: "n" }] }, - userTurn, - ]); - expect(messages.map((message) => message.id)).toEqual(["m2", "m1"]); + test("a call already in AG-UI's shape is left alone", () => { + const already = { + id: "x", + role: "assistant", + toolCalls: [ + { + id: "c1", + type: "function", + function: { name: "computer_click", arguments: "{}" }, + }, + ], + }; + const { messages, unreadable } = readableTurns([already]); + expect(unreadable).toBe(0); + expect(messages[0]).toEqual(already as never); }); - test("a turn that parses keeps the fields the schema does not name", () => { + test("a turn that is genuinely malformed is still refused", () => { /* - * The reason the original object is returned rather than `parsed.data`. Zod strips unknown keys, - * so handing back the parsed copy would make this a silent rewrite of every message that passed - * — dropping whatever the runtime carries that this file has not heard of. + * The guard is not being removed, only taught a second spelling. A tool call with neither shape + * is something no renderer can draw, and letting it through is how one bad turn used to take a + * whole conversation down. */ - const carrying = { ...userTurn, somethingTheRuntimeAdded: "keep me" }; - const { messages } = readableTurns([carrying]); - expect( - (messages[0] as unknown as { somethingTheRuntimeAdded?: string }) - .somethingTheRuntimeAdded, - ).toBe("keep me"); + const nonsense = { id: "y", role: "assistant", toolCalls: [{ id: "c2" }] }; + const { messages, unreadable } = readableTurns([nonsense]); + expect(messages).toHaveLength(0); + expect(unreadable).toBe(1); }); - test("an empty history is not a failure", () => { - expect(readableTurns([])).toEqual({ messages: [], unreadable: 0 }); + test("a mixed array is refused rather than half-translated", () => { + // Guessing at half of it would be this file inventing history rather than reading it. + const mixed = { + id: "z", + role: "assistant", + toolCalls: [ + { id: "a", name: "one", args: "{}" }, + { + id: "b", + type: "function", + function: { name: "two", arguments: "{}" }, + }, + ], + }; + expect(readableTurns([mixed]).unreadable).toBe(1); }); - test("a thread where nothing parses reports every turn", () => { - // The case that must not read as "this conversation is empty": the transcript has nothing to - // draw, so the count is the only thing that tells the person their history is still there. - const { messages, unreadable } = readableTurns([{ nope: true }, 1, null]); - expect(messages).toEqual([]); - expect(unreadable).toBe(3); + test("everything else passes through untouched", () => { + const { messages, unreadable } = readableTurns([userTurn, answer]); + expect(unreadable).toBe(0); + expect(messages[0]).toEqual(userTurn as never); }); }); From 469a38ebc622b25987b59c45c12c0eb31870f47c Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 24 Aug 2026 18:48:40 -0700 Subject: [PATCH 7/9] Show the page a finished turn opened, not the one open now Reopening a conversation made every past turn fetch the screen as it is now, so an answer about Hacker News from an hour ago sat under a picture of whatever the Bot had open since. The frame was live and the caption was not, and the turn read as though it had browsed somewhere it never went. A turn that has finished is history, and history is not polled. It names the page that turn actually left open, which the tool result already carried. Nothing changes while a turn runs: those frames are its own and freeze where it left them. It names the page rather than showing it, because nothing stored the picture and fetching one now would show a different page. Naming it stays true however many times the Bot has browsed since. Driven on EKS: three turns, three different pages, each holding its own across a reload. --- CHANGELOG.md | 14 +++++ app/src/components/computer/computer-view.tsx | 56 ++++++++++++++++++- app/src/lib/copilot/computer-tools.tsx | 34 +++++++++-- 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dc22072..483a8453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,20 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### A finished turn shows the page it opened, not the one open now + +Reopening a conversation made every past turn fetch the screen as it is now, so an answer about +Hacker News from an hour ago sat under a picture of whatever the Bot had open since. The frame was +live and the caption was not, and the turn read as though it had browsed somewhere it never went. + +A turn that has finished is history, and history is not polled. It names the page that turn actually +left open. While a turn is running nothing changes: the frames are its own and freeze where it left +them. + +It names the page rather than showing it, because nothing stored the picture and fetching one now +would show a different page. Naming it is the honest version of the same sentence, and it stays true +however many times the Bot has browsed since. + ### A conversation keeps the browsing that produced its answers Every turn in which a Bot used a tool was disappearing from the transcript on reload. The sentence diff --git a/app/src/components/computer/computer-view.tsx b/app/src/components/computer/computer-view.tsx index 9ba91baa..bf772403 100644 --- a/app/src/components/computer/computer-view.tsx +++ b/app/src/components/computer/computer-view.tsx @@ -18,6 +18,15 @@ function isBlankBrowser(shot: Screenshot): boolean { return url === "" || url === "about:blank"; } +/** The part of a URL worth putting on screen; the whole thing is rarely readable at this size. */ +function hostOf(url: string): string { + try { + return new URL(url).host; + } catch { + return url; + } +} + /** Default browser viewport ratio, reserved before the first screenshot arrives. */ const DEFAULT_ASPECT_RATIO = 1280 / 800; @@ -55,6 +64,15 @@ type Props = { aspectRatio?: number; minWidth?: number; minHeight?: number; + /** + * The page this turn left the browser on, for a turn that has finished. + * + * A conversation is a record, and a record must not change its mind. Without this, reopening a + * conversation made every past turn fetch the screen as it is now, so an answer about Hacker News + * from an hour ago sat under a picture of whatever the Bot has open today. The frame was live, the + * caption was not, and the turn read as though it had browsed somewhere it never went. + */ + page?: { url?: string; title?: string }; }; export function ComputerView({ @@ -64,6 +82,7 @@ export function ComputerView({ aspectRatio = DEFAULT_ASPECT_RATIO, minWidth = DEFAULT_MIN_WIDTH, minHeight = DEFAULT_MIN_HEIGHT, + page, }: Props) { const [shot, setShot] = useState(null); const [problem, setProblem] = useState(null); @@ -92,8 +111,22 @@ export function ComputerView({ /** Force a short watch window after non-Bot actions such as secret entry. */ const watchUntil = useRef(0); + /** + * A finished turn is history, and history is not polled. + * + * While a turn runs, the frames are that turn's own and freeze where it left them, which is right. + * Reopening the conversation later is the case this guards: the component mounts with no frame, + * and fetching one would put today's page under yesterday's answer. It shows the page that turn + * actually left open instead, which is the thing being remembered. + * + * `page` is what marks a turn as settled history rather than one still going, so a caller that + * knows nothing about the page keeps the old behaviour and nothing regresses. + */ + const settled = !active && Boolean(page) && shot === null; + // biome-ignore lint/correctness/useExhaustiveDependencies: `secretPending` intentionally restarts settled polling. useEffect(() => { + if (settled) return; const mine = ++generation.current; let timer: ReturnType; // Consecutive identical frames observed during post-action settling. @@ -141,10 +174,12 @@ export function ComputerView({ generation.current++; clearTimeout(timer); }; - }, [computerId, active, intervalMs, secretPending]); + }, [computerId, active, intervalMs, secretPending, settled]); /** Poll control state independently from screenshot polling so help/secret prompts surface. */ useEffect(() => { + // Nothing is waiting on a turn that has already finished, so nothing needs asking about it. + if (settled) return; let live = true; let timer: ReturnType; const tick = async () => { @@ -228,7 +263,24 @@ export function ComputerView({ : "text-muted-foreground" }`} > - {problem ? ( + {settled ? ( + <> + {/* + What this turn had open, named rather than drawn. + + The picture is gone: nothing stored it, and fetching one now would show a + different page. Naming the page is the honest version of the same sentence, and + it stays true however many times the Bot has browsed since. + */} + {page?.title || "A page"} + {page?.url ? ( + {hostOf(page.url)} + ) : null} + + Opened during this turn. The screen has moved on since. + + + ) : problem ? ( <> You cannot see the screen right now diff --git a/app/src/lib/copilot/computer-tools.tsx b/app/src/lib/copilot/computer-tools.tsx index c4087b49..c4c3275a 100644 --- a/app/src/lib/copilot/computer-tools.tsx +++ b/app/src/lib/copilot/computer-tools.tsx @@ -126,6 +126,9 @@ type ComputerOutcome = { bytes?: number; /** A file read. Named `text` on the way back and `contents` on the way in. */ text?: string; + /** Where a navigation landed, which is what a finished turn's screen tile remembers. */ + url?: string; + title?: string; }; /** @@ -279,11 +282,32 @@ export function ComputerTools() { } : result; }, - render: ({ status }) => ( -
- -
- ), + render: ({ result, status }) => { + /* + * The page this turn left open, so reopening the conversation shows what it browsed rather + * than what the Bot has open now. Only once the turn is finished: while it runs, the live + * frames are its own. + */ + const outcome = status === "complete" ? outcomeOf(result) : {}; + const page = + typeof outcome.url === "string" + ? { + url: outcome.url, + ...(typeof outcome.title === "string" + ? { title: outcome.title } + : {}), + } + : undefined; + return ( +
+ +
+ ); + }, }); useFrontendTool({ From 82eecc2503ed845810aec6ec5b3717ac775c222e Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 24 Aug 2026 19:21:34 -0700 Subject: [PATCH 8/9] Keep the frame a browsing turn ended on Reopening a conversation made every past turn fetch the screen as it is now, so an answer about one page sat under a picture of whatever the Bot had open since. A browsing turn keeps its last frame in computer_turn_frame, filed under the tool call and written once, because a turn that has happened does not happen differently later. Three things had to be true together and each was wrong on its own first. The frame is read at the moment the turn ends, since a short turn finishes before the tile has polled anything. Restoring a kept frame must not make the turn look live again, which the first version did: it counted a turn as history only while it had no picture, so restoring one restarted the polling that then replaced it. And a turn is over when it has a result rather than when its status says so, because a restored tool call arrives with its result in hand and a status that is briefly something else. Found by watching the network on the deployed cluster rather than by reading: two live screenshot reads before the restore, on every reload. --- CHANGELOG.md | 24 +- app/src/components/computer/computer-view.tsx | 100 +- app/src/lib/computers/screen.ts | 43 + app/src/lib/copilot/computer-tools.tsx | 16 +- server/drizzle/0017_turn_frames.sql | 8 + server/drizzle/meta/0017_snapshot.json | 2532 +++++++++++++++++ server/drizzle/meta/_journal.json | 7 + server/src/app.ts | 17 +- server/src/computer/routes.ts | 54 +- server/src/computer/turn-frames.ts | 80 + server/src/db/schema/computer.ts | 35 + server/src/index.ts | 3 + 12 files changed, 2898 insertions(+), 21 deletions(-) create mode 100644 server/drizzle/0017_turn_frames.sql create mode 100644 server/drizzle/meta/0017_snapshot.json create mode 100644 server/src/computer/turn-frames.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 483a8453..43c98e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,16 +11,20 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ### A finished turn shows the page it opened, not the one open now Reopening a conversation made every past turn fetch the screen as it is now, so an answer about -Hacker News from an hour ago sat under a picture of whatever the Bot had open since. The frame was -live and the caption was not, and the turn read as though it had browsed somewhere it never went. - -A turn that has finished is history, and history is not polled. It names the page that turn actually -left open. While a turn is running nothing changes: the frames are its own and freeze where it left -them. - -It names the page rather than showing it, because nothing stored the picture and fetching one now -would show a different page. Naming it is the honest version of the same sentence, and it stays true -however many times the Bot has browsed since. +Hacker News from an hour ago sat under a picture of whatever the Bot had open since. + +A browsing turn now keeps the frame it ended on, in `computer_turn_frame`, filed under the tool call +it belongs to and written once: a turn that has happened does not happen differently later. Reopening +the conversation shows that frame rather than the live screen, and a turn with nothing kept names the +page instead of drawing the wrong one. + +Three things had to be true together, and each was wrong on its own first. The frame is read at the +moment the turn ends, because a short turn finishes before the tile has polled anything and having no +picture in hand is the ordinary case. Restoring a kept frame must not make the turn look live again, +which the first version did: it counted a turn as history only while it had no picture, so restoring +one restarted the polling that then replaced it. And a turn is over when it has a result, not when its +status says so, because a restored tool call arrives with its result already in hand and a status +that is briefly something else. ### A conversation keeps the browsing that produced its answers diff --git a/app/src/components/computer/computer-view.tsx b/app/src/components/computer/computer-view.tsx index bf772403..3c15e7b2 100644 --- a/app/src/components/computer/computer-view.tsx +++ b/app/src/components/computer/computer-view.tsx @@ -7,7 +7,12 @@ import { supplySecret, takeControl, } from "@/lib/computers/control"; -import { readScreenshot, type Screenshot } from "@/lib/computers/screen"; +import { + keepTurnFrame, + readScreenshot, + readTurnFrame, + type Screenshot, +} from "@/lib/computers/screen"; import { LiveScreen } from "./live-screen"; import { ComputerPlaceholder } from "./placeholder"; @@ -73,6 +78,13 @@ type Props = { * caption was not, and the turn read as though it had browsed somewhere it never went. */ page?: { url?: string; title?: string }; + /** + * The tool call this tile belongs to, which is what a kept frame is filed under. + * + * Without it the tile can still name the page; with it, it can show the page. Optional because the + * side panel is not a turn and has nothing to remember. + */ + toolCallId?: string; }; export function ComputerView({ @@ -83,8 +95,12 @@ export function ComputerView({ minWidth = DEFAULT_MIN_WIDTH, minHeight = DEFAULT_MIN_HEIGHT, page, + toolCallId, }: Props) { const [shot, setShot] = useState(null); + /** Read by the keeper below without making every polled frame retrigger it. */ + const shotRef = useRef(null); + shotRef.current = shot; const [problem, setProblem] = useState(null); const [expanded, setExpanded] = useState(false); const [control, setControl] = useState(null); @@ -121,8 +137,84 @@ export function ComputerView({ * * `page` is what marks a turn as settled history rather than one still going, so a caller that * knows nothing about the page keeps the old behaviour and nothing regresses. + * + * DELIBERATELY NOT "AND WE HAVE NO FRAME YET". That is what this said first, and it undid itself: + * restoring the kept frame set the frame, which made the turn stop counting as history, which + * restarted the polling this exists to prevent, which replaced the restored picture with the live + * one. The turn being over is the fact; whether a picture has arrived yet is not. + */ + const settled = !active && Boolean(page); + + /** + * Whether this tile ever watched the turn it belongs to. + * + * The difference between a turn finishing in front of somebody and a conversation being reopened + * later. Both render an inactive tile; only the first may take the live screen as its own, because + * only in the first is the live screen still the page that turn was on. + */ + const watchedItRun = useRef(false); + if (active) watchedItRun.current = true; + + /* + * The frame this turn ended on, filed at the moment it ended and read back when somebody reopens + * the conversation. + * + * A short turn can finish before the tile has polled anything, so having no frame in hand is the + * ordinary case rather than the exception: one is read at completion, which is the last moment the + * live screen and this turn's screen are the same thing. + * + * Kept once and never rewritten. A turn that has happened does not happen differently later, so a + * second visit must not overwrite the picture with whatever the Bot has open by then, which is + * also why this refuses to run at all for a tile that never saw its turn. */ - const settled = !active && Boolean(page) && shot === null; + useEffect(() => { + if (active || !watchedItRun.current || !toolCallId || !page?.url) return; + let current = true; + void (async () => { + const held = shotRef.current?.base64; + const frame = + held ?? (await readScreenshot(computerId)).frame?.base64 ?? null; + if (!current || !frame) return; + await keepTurnFrame(computerId, toolCallId, { + frame, + url: page.url as string, + ...(page.title ? { title: page.title } : {}), + }); + // Shown as well as kept, so the tile that just watched the turn does not fall back to naming + // the page it is holding a picture of. + if (current && !held) { + setShot({ + base64: frame, + width: 0, + height: 0, + capturedAt: "", + url: page.url, + }); + } + })(); + return () => { + current = false; + }; + }, [active, computerId, toolCallId, page?.url, page?.title]); + + /** On reopening, the kept frame rather than the live screen. */ + useEffect(() => { + if (!settled || !toolCallId) return; + let current = true; + void readTurnFrame(computerId, toolCallId).then((kept) => { + if (!current || !kept) return; + setShot({ + base64: kept.frame, + width: 0, + height: 0, + capturedAt: "", + url: kept.url, + }); + }); + return () => { + current = false; + }; + }, [settled, computerId, toolCallId]); // biome-ignore lint/correctness/useExhaustiveDependencies: `secretPending` intentionally restarts settled polling. useEffect(() => { @@ -193,7 +285,7 @@ export function ComputerView({ live = false; clearTimeout(timer); }; - }, [computerId]); + }, [computerId, settled]); // Input forwarding lives in LiveScreen on the socket. // Escape is bound to the window so it works regardless of overlay focus. @@ -263,7 +355,7 @@ export function ComputerView({ : "text-muted-foreground" }`} > - {settled ? ( + {settled && !shot ? ( <> {/* What this turn had open, named rather than drawn. diff --git a/app/src/lib/computers/screen.ts b/app/src/lib/computers/screen.ts index 874a5438..8c987466 100644 --- a/app/src/lib/computers/screen.ts +++ b/app/src/lib/computers/screen.ts @@ -40,3 +40,46 @@ export async function readScreenshot( return { error: unavailable }; } } + +/** The frame a browsing turn ended on, as it was kept. */ +export type TurnFrame = { url: string; title: string | null; frame: string }; + +/** + * Keep the frame this turn ended on. + * + * Fire and forget, and silent on failure. The turn has already happened and its answer is already on + * screen; a conversation must not report an error because the picture of it could not be filed. A + * turn with no stored frame falls back to naming the page, which is the same sentence with less in + * it rather than a broken one. + */ +export async function keepTurnFrame( + computerId: string, + toolCallId: string, + frame: { frame: string; url: string; title?: string }, +): Promise { + try { + await tryClient( + `/api/computers/${computerId}/turn-frames/${encodeURIComponent(toolCallId)}`, + { method: "POST", body: frame }, + ); + } catch { + // See above: nothing here is worth interrupting a conversation for. + } +} + +/** What that turn had on screen, or nothing if it was never kept. */ +export async function readTurnFrame( + computerId: string, + toolCallId: string, +): Promise { + try { + const response = await tryClient( + `/api/computers/${computerId}/turn-frames/${encodeURIComponent(toolCallId)}`, + ); + if (!response.ok) return null; + const body = (await response.json()) as { frame?: TurnFrame | null }; + return body.frame ?? null; + } catch { + return null; + } +} diff --git a/app/src/lib/copilot/computer-tools.tsx b/app/src/lib/copilot/computer-tools.tsx index c4c3275a..8a32d768 100644 --- a/app/src/lib/copilot/computer-tools.tsx +++ b/app/src/lib/copilot/computer-tools.tsx @@ -282,13 +282,22 @@ export function ComputerTools() { } : result; }, - render: ({ result, status }) => { + render: ({ result, status, toolCallId }) => { /* * The page this turn left open, so reopening the conversation shows what it browsed rather * than what the Bot has open now. Only once the turn is finished: while it runs, the live * frames are its own. */ - const outcome = status === "complete" ? outcomeOf(result) : {}; + /* + * A RESULT IS WHAT MAKES A TURN OVER, not the status. + * + * A restored tool call arrives with its result already in hand and a status that is briefly + * something other than complete, so keying on the status alone made every reopened turn look + * like one still running: the tile polled the live screen, put today's page under yesterday's + * answer, and only then restored the frame it should have shown from the start. + */ + const finished = status === "complete" || result !== undefined; + const outcome = finished ? outcomeOf(result) : {}; const page = typeof outcome.url === "string" ? { @@ -302,8 +311,9 @@ export function ComputerTools() {
); diff --git a/server/drizzle/0017_turn_frames.sql b/server/drizzle/0017_turn_frames.sql new file mode 100644 index 00000000..dbe92ebe --- /dev/null +++ b/server/drizzle/0017_turn_frames.sql @@ -0,0 +1,8 @@ +CREATE TABLE "computer_turn_frame" ( + "tool_call_id" text PRIMARY KEY NOT NULL, + "computer_id" text NOT NULL, + "url" text NOT NULL, + "title" text, + "frame" text NOT NULL, + "captured_at" timestamp with time zone DEFAULT now() NOT NULL +); diff --git a/server/drizzle/meta/0017_snapshot.json b/server/drizzle/meta/0017_snapshot.json new file mode 100644 index 00000000..a2034358 --- /dev/null +++ b/server/drizzle/meta/0017_snapshot.json @@ -0,0 +1,2532 @@ +{ + "id": "a4e3f507-e3ff-435c-b828-75fd961ead9f", + "prevId": "0333e21d-0b63-4aac-9234-6ba46382f2b2", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_provider_account_idx": { + "name": "accounts_provider_account_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "agent_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_package_id_deployment_packages_id_fk": { + "name": "agents_package_id_deployment_packages_id_fk", + "tableFrom": "agents", + "tableTo": "deployment_packages", + "columnsFrom": ["package_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_events": { + "name": "audit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_events_created_at_idx": { + "name": "audit_events_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_type_time_idx": { + "name": "audit_events_type_time_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_actor_time_idx": { + "name": "audit_events_actor_time_idx", + "columns": [ + { + "expression": "actor_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_events_target_time_idx": { + "name": "audit_events_target_time_idx", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_agents": { + "name": "channel_agents", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_agents_channel_id_channels_id_fk": { + "name": "channel_agents_channel_id_channels_id_fk", + "tableFrom": "channel_agents", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_agents_agent_id_agents_id_fk": { + "name": "channel_agents_agent_id_agents_id_fk", + "tableFrom": "channel_agents", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_agents_channel_id_agent_id_pk": { + "name": "channel_agents_channel_id_agent_id_pk", + "columns": ["channel_id", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channel_memberships": { + "name": "channel_memberships", + "schema": "", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "channel_memberships_channel_id_channels_id_fk": { + "name": "channel_memberships_channel_id_channels_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "channel_memberships_user_id_users_id_fk": { + "name": "channel_memberships_user_id_users_id_fk", + "tableFrom": "channel_memberships", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_memberships_channel_id_user_id_pk": { + "name": "channel_memberships_channel_id_user_id_pk", + "columns": ["channel_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.channels": { + "name": "channels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suggested_prompts": { + "name": "suggested_prompts", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "allowed_groups": { + "name": "allowed_groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "package_id": { + "name": "package_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "override": { + "name": "override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_message": { + "name": "last_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_message_at": { + "name": "last_message_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_message_agent_id": { + "name": "last_message_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "channels_recent_activity_idx": { + "name": "channels_recent_activity_idx", + "columns": [ + { + "expression": "COALESCE(\"last_message_at\", \"created_at\") DESC", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "channels_package_id_deployment_packages_id_fk": { + "name": "channels_package_id_deployment_packages_id_fk", + "tableFrom": "channels", + "tableTo": "deployment_packages", + "columnsFrom": ["package_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "channels_last_message_agent_id_agents_id_fk": { + "name": "channels_last_message_agent_id_agents_id_fk", + "tableFrom": "channels", + "tableTo": "agents", + "columnsFrom": ["last_message_agent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credentials": { + "name": "credentials", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "credential_kind", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_value": { + "name": "encrypted_value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "credentials_active_key_idx": { + "name": "credentials_active_key_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"credentials\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployment_packages": { + "name": "deployment_packages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "loaded_at": { + "name": "loaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployment_packages_tenant_id_unique": { + "name": "deployment_packages_tenant_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.intelligence_channel_mappings": { + "name": "intelligence_channel_mappings", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "intelligence_channel_mappings_thread_idx": { + "name": "intelligence_channel_mappings_thread_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "intelligence_channel_mappings_user_id_users_id_fk": { + "name": "intelligence_channel_mappings_user_id_users_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "intelligence_channel_mappings_channel_id_channels_id_fk": { + "name": "intelligence_channel_mappings_channel_id_channels_id_fk", + "tableFrom": "intelligence_channel_mappings", + "tableTo": "channels", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "intelligence_channel_mappings_user_id_channel_id_pk": { + "name": "intelligence_channel_mappings_user_id_channel_id_pk", + "columns": ["user_id", "channel_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.revoked_access": { + "name": "revoked_access", + "schema": "", + "columns": { + "email": { + "name": "email", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_providers": { + "name": "sso_providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sso_providers_user_id_users_id_fk": { + "name": "sso_providers_user_id_users_id_fk", + "tableFrom": "sso_providers", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "nullsNotDistinct": false, + "columns": ["provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_pk": { + "name": "user_roles_user_id_role_pk", + "columns": ["user_id", "role"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "groups": { + "name": "groups", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verifications": { + "name": "verifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.action_policy": { + "name": "action_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deny": { + "name": "deny", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "allow": { + "name": "allow", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_snapshot": { + "name": "computer_snapshot", + "schema": "", + "columns": { + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "elements": { + "name": "elements", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "taken_at": { + "name": "taken_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "session": { + "name": "session", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.computer_turn_frame": { + "name": "computer_turn_frame", + "schema": "", + "columns": { + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "computer_id": { + "name": "computer_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "frame": { + "name": "frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_preferences": { + "name": "agent_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_preferences_user_id_users_id_fk": { + "name": "agent_preferences_user_id_users_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_preferences_agent_id_agents_id_fk": { + "name": "agent_preferences_agent_id_agents_id_fk", + "tableFrom": "agent_preferences", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_preferences_user_id_agent_id_pk": { + "name": "agent_preferences_user_id_agent_id_pk", + "columns": ["user_id", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_profiles": { + "name": "agent_profiles", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_description": { + "name": "role_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_seed": { + "name": "avatar_seed", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "agent_visibility", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "callback_token_hash": { + "name": "callback_token_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "callback_token_issued_at": { + "name": "callback_token_issued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_profiles_visibility_deleted_idx": { + "name": "agent_profiles_visibility_deleted_idx", + "columns": [ + { + "expression": "visibility", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_profiles_agent_id_agents_id_fk": { + "name": "agent_profiles_agent_id_agents_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_profiles_owner_user_id_users_id_fk": { + "name": "agent_profiles_owner_user_id_users_id_fk", + "tableFrom": "agent_profiles", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_exclusions": { + "name": "component_exclusions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "withheld_by": { + "name": "withheld_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_exclusions_component_name_components_name_fk": { + "name": "component_exclusions_component_name_components_name_fk", + "tableFrom": "component_exclusions", + "tableTo": "components", + "columnsFrom": ["component_name"], + "columnsTo": ["name"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "component_exclusions_agent_id_agents_id_fk": { + "name": "component_exclusions_agent_id_agents_id_fk", + "tableFrom": "component_exclusions", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_exclusions_component_name_agent_id_pk": { + "name": "component_exclusions_component_name_agent_id_pk", + "columns": ["component_name", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.component_functions": { + "name": "component_functions", + "schema": "", + "columns": { + "component_name": { + "name": "component_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "function_name": { + "name": "function_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "component_functions_component_name_components_name_fk": { + "name": "component_functions_component_name_components_name_fk", + "tableFrom": "component_functions", + "tableTo": "components", + "columnsFrom": ["component_name"], + "columnsTo": ["name"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "component_functions_component_name_function_name_pk": { + "name": "component_functions_component_name_function_name_pk", + "columns": ["component_name", "function_name"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.components": { + "name": "components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_servers": { + "name": "mcp_servers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vendor": { + "name": "vendor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'first-party'" + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "tools_refreshed_at": { + "name": "tools_refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "added_by": { + "name": "added_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_servers_credential_id_credentials_id_fk": { + "name": "mcp_servers_credential_id_credentials_id_fk", + "tableFrom": "mcp_servers", + "tableTo": "credentials", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_tools": { + "name": "mcp_tools", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "input_schema": { + "name": "input_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_tools_server_id_mcp_servers_id_fk": { + "name": "mcp_tools_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_tools", + "tableTo": "mcp_servers", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_tools_server_id_name_pk": { + "name": "mcp_tools_server_id_name_pk", + "columns": ["server_id", "name"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_user_credentials": { + "name": "mcp_user_credentials", + "schema": "", + "columns": { + "server_id": { + "name": "server_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mcp_user_credentials_user_idx": { + "name": "mcp_user_credentials_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_user_credentials_server_id_mcp_servers_id_fk": { + "name": "mcp_user_credentials_server_id_mcp_servers_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "mcp_servers", + "columnsFrom": ["server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_user_id_users_id_fk": { + "name": "mcp_user_credentials_user_id_users_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_user_credentials_credential_id_credentials_id_fk": { + "name": "mcp_user_credentials_credential_id_credentials_id_fk", + "tableFrom": "mcp_user_credentials", + "tableTo": "credentials", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "mcp_user_credentials_server_id_user_id_pk": { + "name": "mcp_user_credentials_server_id_user_id_pk", + "columns": ["server_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_grants": { + "name": "plugin_grants", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_by": { + "name": "granted_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "plugin_grants_agent_idx": { + "name": "plugin_grants_agent_idx", + "columns": [ + { + "expression": "agent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "plugin_grants_agent_id_agents_id_fk": { + "name": "plugin_grants_agent_id_agents_id_fk", + "tableFrom": "plugin_grants", + "tableTo": "agents", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "plugin_grants_kind_ref_agent_id_pk": { + "name": "plugin_grants_kind_ref_agent_id_pk", + "columns": ["kind", "ref", "agent_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sandboxed_components": { + "name": "sandboxed_components", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "draft_description": { + "name": "draft_description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_html": { + "name": "draft_html", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_css": { + "name": "draft_css", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_js_functions": { + "name": "draft_js_functions", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "draft_argument_schema": { + "name": "draft_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "published_description": { + "name": "published_description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_html": { + "name": "published_html", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_css": { + "name": "published_css", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_js_functions": { + "name": "published_js_functions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_argument_schema": { + "name": "published_argument_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "sample_arguments": { + "name": "sample_arguments", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "authored_by": { + "name": "authored_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skill_tools": { + "name": "skill_tools", + "schema": "", + "columns": { + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref": { + "name": "ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "declared_by": { + "name": "declared_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skill_tools_ref_idx": { + "name": "skill_tools_ref_idx", + "columns": [ + { + "expression": "ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skill_tools_skill_id_skills_id_fk": { + "name": "skill_tools_skill_id_skills_id_fk", + "tableFrom": "skill_tools", + "tableTo": "skills", + "columnsFrom": ["skill_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "skill_tools_skill_id_ref_pk": { + "name": "skill_tools_skill_id_ref_pk", + "columns": ["skill_id", "ref"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.skills": { + "name": "skills", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'yours'" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "skills_slug_key": { + "name": "skills_slug_key", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "skills_owner_idx": { + "name": "skills_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "skills_owner_user_id_users_id_fk": { + "name": "skills_owner_user_id_users_id_fk", + "tableFrom": "skills", + "tableTo": "users", + "columnsFrom": ["owner_user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.work_items": { + "name": "work_items", + "schema": "", + "columns": { + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_at": { + "name": "run_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "work_items_claimable_idx": { + "name": "work_items_claimable_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "work_items_kind_key_pk": { + "name": "work_items_kind_key_pk", + "columns": ["kind", "key"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.agent_type": { + "name": "agent_type", + "schema": "public", + "values": ["built_in", "remote_ag_ui"] + }, + "public.credential_kind": { + "name": "credential_kind", + "schema": "public", + "values": [ + "model", + "connector", + "agent", + "mcp", + "mcp_oauth_client", + "mcp_user_token" + ] + }, + "public.role": { + "name": "role", + "schema": "public", + "values": ["admin", "user"] + }, + "public.agent_visibility": { + "name": "agent_visibility", + "schema": "public", + "values": ["public", "private"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/server/drizzle/meta/_journal.json b/server/drizzle/meta/_journal.json index ebd2290e..0740a29f 100644 --- a/server/drizzle/meta/_journal.json +++ b/server/drizzle/meta/_journal.json @@ -120,6 +120,13 @@ "when": 1787602199792, "tag": "0016_durable_work", "breakpoints": true + }, + { + "idx": 17, + "version": "7", + "when": 1787622676444, + "tag": "0017_turn_frames", + "breakpoints": true } ] } diff --git a/server/src/app.ts b/server/src/app.ts index edd47dff..d1f27286 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -5,8 +5,6 @@ import { authoriseAgentCall } from "./agents/callback-token"; import type { BotAccessCheck } from "./agents/profile-policy"; import type { AgentProfileStore } from "./agents/profile-store"; import { createAgentRoutes } from "./agents/routes"; -import { createRoutingRoutes } from "./routing/routes"; -import type { IntentRouter } from "./routing/classify"; import { type AuditReader, type AuditStore, @@ -34,6 +32,7 @@ import type { ComponentStore } from "./components/store"; import type { ComputerGateway } from "./computer/gateway"; import type { PolicyStore } from "./computer/policy-store"; import { createComputerRoutes } from "./computer/routes"; +import type { TurnFrameStore } from "./computer/turn-frames"; import { configuredAuthProviders, type DeploymentConfig } from "./config"; import type { CredentialAdminService, CredentialInput } from "./credentials"; import { createIntelligenceClient } from "./intelligence-client"; @@ -41,6 +40,8 @@ import type { PeopleStore } from "./people/store"; import { createPluginRoutes } from "./plugins/routes"; import type { PluginStore } from "./plugins/store"; import { REFUSAL_MARKER } from "./plugins/tools"; +import type { IntentRouter } from "./routing/classify"; +import { createRoutingRoutes } from "./routing/routes"; import type { PackageStatusReader } from "./tenant-package"; /** @@ -155,6 +156,17 @@ export function createApp( * the default coworker, which is exactly the failsafe the router itself falls back to. */ intentRouter?: IntentRouter, + /** + * Where the frame a browsing turn ended on is kept. + * + * Appended last on purpose: these are positional, so inserting one anywhere else silently + * shifts every existing call site's arguments by one. + * + * Absent leaves the transcript working and past turns without a picture, which is the correct + * degraded behaviour: a conversation that cannot show what it saw is better than one that shows + * the wrong thing. + */ + turnFrames?: TurnFrameStore, ) { const app = new Hono<{ Variables: AppVariables }>(); @@ -630,6 +642,7 @@ export function createApp( computerPolicy, requireUser, canUseBot, + turnFrames, ), ); } diff --git a/server/src/computer/routes.ts b/server/src/computer/routes.ts index e8a88be6..86f29b90 100644 --- a/server/src/computer/routes.ts +++ b/server/src/computer/routes.ts @@ -3,20 +3,21 @@ import { Hono } from "hono"; import type { BotAccessCheck } from "../agents/profile-policy"; import type { AppVariables } from "../auth/guards"; import { requireAdmin } from "../auth/guards"; +import { DEPLOYMENT_ROUTES } from "./deployment-routes"; import { type ActionActor, ActionRefusedError, type ComputerGateway, ComputerUnavailableError, ElementNotFoundError, - NavigationRefusedError, HumanHasControlError, + NavigationRefusedError, StaleSnapshotError, WorkspaceRefusedError, WorkspaceRequestError, } from "./gateway"; -import { DEPLOYMENT_ROUTES } from "./deployment-routes"; import { type PolicyStore, parseActionPolicy } from "./policy-store"; +import type { TurnFrameStore } from "./turn-frames"; /** * The Bot computer's surface, behind the same session guard as every other API route. @@ -38,6 +39,8 @@ export function createComputerRoutes( * deployment cannot be wired up without an answer to it. */ canUseBot: BotAccessCheck, + /** Where a finished turn's screenshot is kept. Absent leaves the transcript as it was. */ + turnFrames?: TurnFrameStore, ) { const routes = new Hono<{ Variables: AppVariables }>(); @@ -361,6 +364,53 @@ export function createComputerRoutes( }); /** The Bot's files. Through the gateway, like every other acting call. */ + /* + * The frame a browsing turn ended on, kept so the transcript can show it later. + * + * Written by the surface rather than by the gateway, because the gateway does not know which tool + * call it is serving: the id belongs to the conversation. Not an acting route, so it does not go + * through `act`: nothing is decided and nothing reaches the computer, it only stores a picture the + * caller already had on screen. + */ + routes.post("/:botId/turn-frames/:toolCallId", async (context) => { + if (!turnFrames) return context.json({ ok: true }); + const botId = context.req.param("botId"); + const toolCallId = context.req.param("toolCallId"); + const body = await context.req.json().catch(() => null); + if ( + typeof body?.frame !== "string" || + typeof body?.url !== "string" || + !body.frame || + !body.url + ) { + return context.json({ error: "A frame and a url are required." }, 400); + } + try { + await turnFrames.save({ + toolCallId, + computerId: botId, + url: body.url, + ...(typeof body.title === "string" ? { title: body.title } : {}), + frame: body.frame, + }); + } catch (error) { + return context.json( + { error: error instanceof Error ? error.message : "Not stored." }, + 413, + ); + } + return context.json({ ok: true }); + }); + + routes.get("/:botId/turn-frames/:toolCallId", async (context) => { + if (!turnFrames) return context.json({ frame: null }); + const stored = await turnFrames.load( + context.req.param("toolCallId"), + context.req.param("botId"), + ); + return context.json({ frame: stored }); + }); + routes.post("/:botId/files/list", (context) => act(context, (botId, actor, body) => gateway.listFiles(botId, actor, { diff --git a/server/src/computer/turn-frames.ts b/server/src/computer/turn-frames.ts new file mode 100644 index 00000000..fe0907d9 --- /dev/null +++ b/server/src/computer/turn-frames.ts @@ -0,0 +1,80 @@ +/** + * What a Bot's screen looked like when one turn finished with it. + * + * Written once when a browsing turn ends and read back when somebody reopens the conversation. The + * transcript used to fetch the live screen for every past turn, so an answer about one page sat under + * a picture of whichever page the Bot had open by the time it was read back. + */ +import { and, eq } from "drizzle-orm"; +import type { Database } from "../db/client"; +import { computerTurnFrame } from "../db/schema"; + +/** + * A screenshot, and the ceiling on one. + * + * Generous enough for a full page at the sizes a computer runs, small enough that nothing can push + * megabytes into the transcript by calling this in a loop. Refused at the boundary rather than + * truncated, because half a PNG is not a smaller picture, it is a broken one. + */ +const MAX_FRAME_BYTES = 4 * 1024 * 1024; + +export type TurnFrameStore = { + save: (frame: { + toolCallId: string; + computerId: string; + url: string; + title?: string; + frame: string; + }) => Promise; + load: ( + toolCallId: string, + computerId: string, + ) => Promise<{ url: string; title: string | null; frame: string } | null>; +}; + +export function createTurnFrameStore(database: Database): TurnFrameStore { + return { + async save(input) { + if (input.frame.length > MAX_FRAME_BYTES) { + throw new Error("That screenshot is too large to keep."); + } + await database + .insert(computerTurnFrame) + .values({ + toolCallId: input.toolCallId, + computerId: input.computerId, + url: input.url, + ...(input.title ? { title: input.title } : {}), + frame: input.frame, + }) + /* + * Nothing on conflict. A turn that has happened does not happen differently later, and the + * first frame written for a tool call is the one that turn ended on. A retry or a second + * render must not overwrite it with whatever is on screen by then. + */ + .onConflictDoNothing(); + }, + + async load(toolCallId, computerId) { + const [row] = await database + .select({ + url: computerTurnFrame.url, + title: computerTurnFrame.title, + frame: computerTurnFrame.frame, + }) + .from(computerTurnFrame) + /* + * Both, always. The tool call id alone is unique, but a caller who may reach one Bot must not + * be able to read a frame from another by guessing an id: the Bot in the path is what the + * route already checked, so it is what this is keyed on too. + */ + .where( + and( + eq(computerTurnFrame.toolCallId, toolCallId), + eq(computerTurnFrame.computerId, computerId), + ), + ); + return row ?? null; + }, + }; +} diff --git a/server/src/db/schema/computer.ts b/server/src/db/schema/computer.ts index 87b508d5..dc86fcca 100644 --- a/server/src/db/schema/computer.ts +++ b/server/src/db/schema/computer.ts @@ -80,3 +80,38 @@ export const computerSnapshot = pgTable("computer_snapshot", { */ session: text("session"), }); + +/** + * What a Bot's screen looked like when one turn finished with it. + * + * A conversation is a record, and a record must not change its mind. The transcript used to fetch + * the live screen for every past turn, so an answer about one page sat under a picture of whichever + * page the Bot had open by the time somebody read it back: the frame was true and the turn it sat in + * was not. + * + * KEYED ON THE TOOL CALL, not on the computer, because that is the thing being remembered. One row + * per browsing turn, written once when the turn ends and never updated: a turn that has happened + * does not happen differently later. + * + * The image and not only the address, because "here is where it went" is a weaker sentence than the + * page itself, and the picture is the whole reason the tile exists. + */ +export const computerTurnFrame = pgTable("computer_turn_frame", { + /** The tool call this frame belongs to, which is unique across every conversation. */ + toolCallId: text("tool_call_id").primaryKey(), + /** Whose computer it was, so a frame can be removed with the Bot it belonged to. */ + computerId: text("computer_id").notNull(), + /** The page, for the caption and for a frame that fails to decode. */ + url: text("url").notNull(), + title: text("title"), + /** + * The frame itself, base64 PNG. + * + * Bounded by the route that writes it rather than by the column, because the useful limit is "a + * screenshot" and the honest failure is a refusal at the boundary rather than a database error. + */ + frame: text("frame").notNull(), + capturedAt: timestamp("captured_at", { withTimezone: true }) + .notNull() + .defaultNow(), +}); diff --git a/server/src/index.ts b/server/src/index.ts index 4b7f2bf6..ecdd6b8f 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -32,6 +32,7 @@ import { describeComputerIsolation, } from "./computer/provider"; import { createSnapshotStore } from "./computer/snapshot-store"; +import { createTurnFrameStore } from "./computer/turn-frames"; import { loadConfig } from "./config"; import { type IdentifyActor, @@ -541,6 +542,8 @@ const app = createApp( identityProviderStore, // Chooses the coworker for an untagged message, on the deployment's own model and key. intentRouter, + // What a browsing turn's screen looked like when it finished, so the transcript can show it later. + createTurnFrameStore(database), ); /** From f6dc34d0463c565fbce868e02c6733a32699b155 Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 24 Aug 2026 21:34:53 -0700 Subject: [PATCH 9/9] Keep a turn's frame only when it is a frame of that turn's page The capture ran at the end of a turn and took whatever the screen showed then. That is usually right and sometimes badly wrong: the same computer is driven by other conversations, a resumed one starts blank, and a short turn finishes before the tile has polled anything. So an answer about one page could be filed with a picture of another, which is worse than having no picture at all. A frame is now kept only when its own url is the page the turn opened. Unknown counts as no match, because storing on unknown is how the wrong picture gets kept. Also folds the restore and the capture into one effect asked in order: what is stored first, the live screen only if nothing is. Two effects racing is what made a reopened turn restore the right frame and then overwrite it with a fresh screenshot one render later, which the console showed plainly once I stopped guessing and logged it. --- app/src/components/computer/computer-view.tsx | 209 +++++++++++++----- 1 file changed, 150 insertions(+), 59 deletions(-) diff --git a/app/src/components/computer/computer-view.tsx b/app/src/components/computer/computer-view.tsx index 3c15e7b2..074e181b 100644 --- a/app/src/components/computer/computer-view.tsx +++ b/app/src/components/computer/computer-view.tsx @@ -32,6 +32,63 @@ function hostOf(url: string): string { } } +/** + * What each finished turn opened, and the frame it ended on, kept outside any component. + * + * MODULE SCOPE, BECAUSE THE TILE DOES NOT SURVIVE. A transcript re-renders freely and remounts the + * tiles in it, and anything held in component state goes with it: the fresh mount has no page yet, + * behaves for one render like a live turn, and reaches for the live screen. Keyed on the tool call, + * which is the identity of the turn rather than of the component drawing it. + * + * Bounded, because a long conversation is a lot of screenshots. Oldest out first, and a turn whose + * frame has been dropped falls back to naming its page. + */ +type RememberedTurn = { + page?: { url?: string; title?: string }; + frame?: { base64: string; url: string }; +}; +const REMEMBERED_TURNS = new Map(); +const MAX_REMEMBERED_TURNS = 40; + +function rememberTurn(toolCallId: string, patch: RememberedTurn): void { + const existing = REMEMBERED_TURNS.get(toolCallId) ?? {}; + /* + * A FRAME IS WRITTEN ONCE, which is what the server's own insert says and what this has to agree + * with. Letting a later write win is exactly what went wrong: the tile restored the right frame and + * then replaced it, one render later, with a screenshot of whatever the Bot had open by then. + */ + const merged: RememberedTurn = { ...existing, ...patch }; + if (existing.frame) merged.frame = existing.frame; + REMEMBERED_TURNS.delete(toolCallId); + REMEMBERED_TURNS.set(toolCallId, merged); + while (REMEMBERED_TURNS.size > MAX_REMEMBERED_TURNS) { + const oldest = REMEMBERED_TURNS.keys().next().value; + if (oldest === undefined) break; + REMEMBERED_TURNS.delete(oldest); + } +} + +/** + * Whether a frame is showing the page a turn opened. + * + * Compared without the trailing slash a browser adds and without the fragment, which are the two + * ways the same page reports two spellings of itself. A frame whose page cannot be read is not a + * match: unknown is not the same as equal, and storing on unknown is how the wrong picture gets kept. + */ +function samePage(frameUrl: string | undefined, pageUrl: string): boolean { + if (!frameUrl) return false; + const tidy = (value: string) => { + try { + const parsed = new URL(value); + parsed.hash = ""; + return parsed.toString().replace(/\/$/, ""); + } catch { + return value.replace(/\/$/, ""); + } + }; + return tidy(frameUrl) === tidy(pageUrl); +} + /** Default browser viewport ratio, reserved before the first screenshot arrives. */ const DEFAULT_ASPECT_RATIO = 1280 / 800; @@ -143,78 +200,100 @@ export function ComputerView({ * restarted the polling this exists to prevent, which replaced the restored picture with the live * one. The turn being over is the fact; whether a picture has arrived yet is not. */ - const settled = !active && Boolean(page); - - /** - * Whether this tile ever watched the turn it belongs to. + /* + * The page this turn opened, remembered rather than re-derived. * - * The difference between a turn finishing in front of somebody and a conversation being reopened - * later. Both render an inactive tile; only the first may take the live screen as its own, because - * only in the first is the live screen still the page that turn was on. + * The prop is rebuilt from the tool result on every render, and a restored turn's result flickers + * in and out of hand while the transcript settles, so the tile kept deciding it was live again. A + * turn that has opened a page has opened it; nothing later makes that untrue. */ - const watchedItRun = useRef(false); - if (active) watchedItRun.current = true; + if (toolCallId && page?.url) rememberTurn(toolCallId, { page }); + const knownPage = + page?.url !== undefined + ? page + : toolCallId + ? REMEMBERED_TURNS.get(toolCallId)?.page + : undefined; + const keptFrame = toolCallId + ? (REMEMBERED_TURNS.get(toolCallId)?.frame ?? null) + : null; + /** Bumped when a frame arrives, because the store it lands in is not React state. */ + const [, setFrameArrived] = useState(0); + + const settled = !active && Boolean(knownPage); + console.info( + "[tile]", + JSON.stringify({ + toolCallId: toolCallId ?? null, + active, + pageUrl: page?.url ?? null, + knownUrl: knownPage?.url ?? null, + settled, + kept: keptFrame?.base64.length ?? null, + shot: shot?.base64.length ?? null, + }), + ); /* - * The frame this turn ended on, filed at the moment it ended and read back when somebody reopens - * the conversation. + * The frame this turn ended on: whatever was filed for it, and only failing that, the screen as it + * is right now. * - * A short turn can finish before the tile has polled anything, so having no frame in hand is the - * ordinary case rather than the exception: one is read at completion, which is the last moment the - * live screen and this turn's screen are the same thing. + * ONE EFFECT, ASKED IN THAT ORDER, because two of them racing is what went wrong. A reopened tile + * and one that has just watched its turn end are indistinguishable from inside the component: both + * are inactive with a result in hand, and both render as live for one frame first. Two effects, one + * restoring and one capturing, therefore both ran on a reopened turn, and the capture replaced the + * frame the restore had just put there with a fresh screenshot of whatever the Bot has open now. * - * Kept once and never rewritten. A turn that has happened does not happen differently later, so a - * second visit must not overwrite the picture with whatever the Bot has open by then, which is - * also why this refuses to run at all for a tile that never saw its turn. + * Asking what is stored before capturing anything makes the difference stop mattering: a reopened + * turn finds a frame and never reaches for the live screen at all. */ useEffect(() => { - if (active || !watchedItRun.current || !toolCallId || !page?.url) return; + if (active || !toolCallId || !knownPage?.url) return; + if (REMEMBERED_TURNS.get(toolCallId)?.frame) return; let current = true; + void (async () => { - const held = shotRef.current?.base64; - const frame = - held ?? (await readScreenshot(computerId)).frame?.base64 ?? null; - if (!current || !frame) return; + const stored = await readTurnFrame(computerId, toolCallId); + if (!current) return; + if (stored) { + rememberTurn(toolCallId, { + frame: { base64: stored.frame, url: stored.url }, + }); + setFrameArrived((n) => n + 1); + return; + } + + /* + * Nothing filed, so this turn ended a moment ago and the live screen may still be its own. + * + * MAY. A short turn can finish before the tile has polled anything, and by the time this asks, + * the browser can already be somewhere else: another turn in another conversation drives the + * same computer, and a resumed one starts blank. So the frame is only this turn's if it is + * showing this turn's page, and a frame that is not is not stored at all. Better a turn that + * names the page it opened than one that shows a picture of somewhere it never went, which is + * what the first version of this did. + */ + const url = knownPage.url as string; + const held = shotRef.current; + const candidate = + held ?? (await readScreenshot(computerId)).frame ?? null; + if (!current || !candidate) return; + if (!samePage(candidate.url, url)) return; + const frame = candidate.base64; await keepTurnFrame(computerId, toolCallId, { frame, - url: page.url as string, - ...(page.title ? { title: page.title } : {}), + url, + ...(knownPage.title ? { title: knownPage.title } : {}), }); - // Shown as well as kept, so the tile that just watched the turn does not fall back to naming - // the page it is holding a picture of. - if (current && !held) { - setShot({ - base64: frame, - width: 0, - height: 0, - capturedAt: "", - url: page.url, - }); - } + if (!current) return; + rememberTurn(toolCallId, { frame: { base64: frame, url } }); + setFrameArrived((n) => n + 1); })(); - return () => { - current = false; - }; - }, [active, computerId, toolCallId, page?.url, page?.title]); - /** On reopening, the kept frame rather than the live screen. */ - useEffect(() => { - if (!settled || !toolCallId) return; - let current = true; - void readTurnFrame(computerId, toolCallId).then((kept) => { - if (!current || !kept) return; - setShot({ - base64: kept.frame, - width: 0, - height: 0, - capturedAt: "", - url: kept.url, - }); - }); return () => { current = false; }; - }, [settled, computerId, toolCallId]); + }, [active, computerId, toolCallId, knownPage?.url, knownPage?.title]); // biome-ignore lint/correctness/useExhaustiveDependencies: `secretPending` intentionally restarts settled polling. useEffect(() => { @@ -299,7 +378,7 @@ export function ComputerView({ }, [expanded]); // Always render the card frame; help/secret controls live below the conditional picture. - const blankBrowser = shot ? isBlankBrowser(shot) : false; + const blankBrowser = !settled && shot ? isBlankBrowser(shot) : false; /* * Sized from the ratio, never from the payload, so the frame is identical while a screen is @@ -314,12 +393,24 @@ export function ComputerView({ const frameStyle = blankBrowser ? { minWidth } : { aspectRatio, minWidth, minHeight }; + /* + * What this tile draws: the kept frame for a turn that is over, the live one while it runs. + * + * A finished turn never draws `shot`. It may hold one, caught in the render between mounting and + * its result arriving, and that frame is of whatever the Bot has open now rather than of this turn. + */ + const drawn = settled + ? keptFrame + : shot + ? { base64: shot.base64, url: shot.url ?? "" } + : null; + /** Blank browser placeholders should not be opened as readable screens. */ - const showScreen = shot !== null && !blankBrowser; + const showScreen = drawn !== null && !blankBrowser; const polledScreen = showScreen ? ( What the assistant is looking at{page?.title || "A page"} - {page?.url ? ( - {hostOf(page.url)} + {knownPage?.url ? ( + {hostOf(knownPage.url)} ) : null} Opened during this turn. The screen has moved on since.