diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index a69c83cd37..f6f1e519d4 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -31,15 +31,18 @@ import { assertSplitRealtimeInterlock, } from "./v3/runOpsMigration/splitMode.server"; import { computeRunOpsSplitReadEnabled } from "./v3/runOpsMigration/runOpsSplitReadGate"; -import { assertControlPlaneCoresidencyAdvisory } from "./v3/runOpsMigration/controlPlaneCoresidencySentinel.server"; -import { DATASOURCE_CONTEXT_KEY, startActiveSpan } from "./v3/tracer.server"; +import { resolveRunOpsPoolKnobs } from "./v3/runOpsPoolKnobs.server"; +import { buildRunOpsShardTable } from "./v3/runOpsShardTable"; import { + resolveShardResilience, controlPlaneTransactionResilience, registerTransactionResilience, resilienceForClient, runOpsLegacyTransactionResilience, runOpsTransactionResilience, } from "./v3/transactionResilience.server"; +import { assertControlPlaneCoresidencyAdvisory } from "./v3/runOpsMigration/controlPlaneCoresidencySentinel.server"; +import { DATASOURCE_CONTEXT_KEY, startActiveSpan } from "./v3/tracer.server"; import type { Span } from "@opentelemetry/api"; import { context, trace } from "@opentelemetry/api"; import { queryPerformanceMonitor } from "./utils/queryPerformanceMonitor.server"; @@ -275,10 +278,19 @@ export const webhookReplica: WebhookReplicaDatabase = singleton("webhookReplica" type RunOpsClients = { writer: PrismaClient; replica: PrismaReplicaClient }; type NewRunOpsClients = { writer: RunOpsPrismaClient; replica: RunOpsPrismaClient }; +type ShardTopologyDescriptor = { + key: string; + url?: string; + replicaUrl?: string; + aliasOf?: "new"; +}; export type RunOpsTopology = { newRunOps: NewRunOpsClients; legacyRunOps: RunOpsClients; controlPlane: RunOpsClients; + // One client pair per gen-2 shard descriptor. Empty unless RUN_OPS_SHARDS is configured. An + // aliasOf:"new" descriptor maps to the newRunOps pair BY REFERENCE (no new pool). + shards: Map; }; export type SelectRunOpsTopologyConfig = { splitEnabled: boolean; @@ -288,6 +300,7 @@ export type SelectRunOpsTopologyConfig = { newReplicaUrl?: string; // When true, legacy reuses the control-plane client instead of opening its own pool. Defaults to false. legacySharesControlPlane?: boolean; + shards?: ShardTopologyDescriptor[]; }; export type RunOpsClientBuilders = { controlPlane: RunOpsClients; @@ -297,6 +310,10 @@ export type RunOpsClientBuilders = { // RunOpsPrismaClient double-cast needed): the legacy DB carries the full control-plane schema. buildLegacyWriter: (url: string, clientType: string) => PrismaClient; buildLegacyReplica: (url: string, clientType: string) => PrismaReplicaClient; + // Receive the whole descriptor so the singleton can resolve per-shard knobs and resilience by key. + // Optional so the existing test literals (which build no shards) need no change. + buildShardWriter?: (shard: ShardTopologyDescriptor) => RunOpsPrismaClient; + buildShardReplica?: (shard: ShardTopologyDescriptor) => RunOpsPrismaClient; }; // Pure run-ops client selector. No env, no isSplitEnabled() — those @@ -315,11 +332,11 @@ export function selectRunOpsTopology( }; if (!config.splitEnabled) { - return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane }; + return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane, shards: new Map() }; } if (!config.legacyUrl || !config.newUrl) { - return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane }; + return { newRunOps: cpFallback, legacyRunOps: controlPlane, controlPlane, shards: new Map() }; } // Same-DB legacy reuses the control-plane pool; only build a separate pool once the DSNs diverge. @@ -338,12 +355,28 @@ export function selectRunOpsTopology( const newReplica: RunOpsPrismaClient = config.newReplicaUrl ? builders.buildNewReplica(config.newReplicaUrl, "run-ops-replica") : newWriter; + const newRunOps: NewRunOpsClients = { writer: newWriter, replica: newReplica }; + + const shards = new Map(); + for (const shard of config.shards ?? []) { + if (shard.aliasOf === "new") { + // Aliased: share the new store's clients by reference. No builder, no new pool — the soak path. + shards.set(shard.key, newRunOps); + continue; + } + if (!shard.url || !builders.buildShardWriter || !builders.buildShardReplica) { + throw new Error( + `selectRunOpsTopology: shard "${shard.key}" needs a url and shard builders when not aliased` + ); + } + const shardWriter = builders.buildShardWriter(shard); + const shardReplica: RunOpsPrismaClient = shard.replicaUrl + ? builders.buildShardReplica(shard) + : shardWriter; + shards.set(shard.key, { writer: shardWriter, replica: shardReplica }); + } - return { - newRunOps: { writer: newWriter, replica: newReplica }, - legacyRunOps, - controlPlane, - }; + return { newRunOps, legacyRunOps, controlPlane, shards }; } // The env-bound run-ops topology singleton. The split decision uses @@ -376,6 +409,17 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { ); } + const newPoolKnobs = resolveRunOpsPoolKnobs("new"); + const shardDescriptorsByKey = new Map(env.RUN_OPS_SHARDS.map((d) => [d.key, d])); + + // Boot table: emit ONLY when shards are configured, so the inert (RUN_OPS_SHARDS unset) merge adds + // no new log output. The fingerprint is an address, not an identity claim (see runOpsAddressFingerprint). + if (env.RUN_OPS_SHARDS.length > 0) { + logger.info("run-ops shard topology (fingerprint is an address, NOT an identity claim)", { + shards: buildRunOpsShardTable(env.RUN_OPS_SHARDS), + }); + } + return selectRunOpsTopology( { splitEnabled, @@ -384,6 +428,12 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { newUrl, newReplicaUrl: env.RUN_OPS_DATABASE_READ_REPLICA_URL, legacySharesControlPlane, + shards: env.RUN_OPS_SHARDS.map((d) => ({ + key: d.key, + url: d.url, + replicaUrl: d.replicaUrl, + aliasOf: d.aliasOf, + })), }, { controlPlane: { writer: prisma, replica: $replica }, @@ -392,10 +442,14 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { captureInfraErrorsRunOps( tagDatasourceRunOps( "run-ops-writer", - buildRunOpsWriterClient({ + buildRunOpsClient({ url, clientType, - useDriverAdapter: env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1", + role: "writer", + connectionLimit: newPoolKnobs.connectionLimit, + poolTimeout: newPoolKnobs.writerPoolTimeout, + connectTimeout: newPoolKnobs.writerConnectionTimeout, + useDriverAdapter: newPoolKnobs.writerDriverAdapter, }) ) ), @@ -409,10 +463,14 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { captureInfraErrorsRunOps( tagDatasourceRunOps( "run-ops-replica", - buildRunOpsReplicaClient({ + buildRunOpsClient({ url, clientType, - useDriverAdapter: env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + role: "replica", + connectionLimit: newPoolKnobs.replicaConnectionLimit, + poolTimeout: newPoolKnobs.replicaPoolTimeout, + connectTimeout: newPoolKnobs.replicaConnectionTimeout, + useDriverAdapter: newPoolKnobs.replicaDriverAdapter, }) ) ) @@ -450,6 +508,50 @@ const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { ) ) ), + // A gen-2 shard is a dedicated run-ops DB, so it mirrors buildNewWriter/buildNewReplica: same + // client class, same wrapper stack, its OWN resilience budget, and the "new"-role pool knobs + // merged with the descriptor's per-shard overrides. Shards share the run-ops datasource tag. + buildShardWriter: (shard) => { + const descriptor = shardDescriptorsByKey.get(shard.key); + const knobs = resolveRunOpsPoolKnobs("new", descriptor?.knobs); + return registerTransactionResilience( + captureInfraErrorsRunOps( + tagDatasourceRunOps( + "run-ops-writer", + buildRunOpsClient({ + url: shard.url!, + clientType: `run-ops-shard-${shard.key}-writer`, + role: "writer", + connectionLimit: knobs.connectionLimit, + poolTimeout: knobs.writerPoolTimeout, + connectTimeout: knobs.writerConnectionTimeout, + useDriverAdapter: knobs.writerDriverAdapter, + }) + ) + ), + resolveShardResilience(shard.key, descriptor?.knobs) + ); + }, + buildShardReplica: (shard) => { + const descriptor = shardDescriptorsByKey.get(shard.key); + const knobs = resolveRunOpsPoolKnobs("new", descriptor?.knobs); + return markReadReplicaClient( + captureInfraErrorsRunOps( + tagDatasourceRunOps( + "run-ops-replica", + buildRunOpsClient({ + url: shard.replicaUrl!, + clientType: `run-ops-shard-${shard.key}-replica`, + role: "replica", + connectionLimit: knobs.replicaConnectionLimit, + poolTimeout: knobs.replicaPoolTimeout, + connectTimeout: knobs.replicaConnectionTimeout, + useDriverAdapter: knobs.replicaDriverAdapter, + }) + ) + ) + ); + }, } ); }); @@ -475,6 +577,17 @@ export const runOpsLegacyPrismaClient: RunOpsPrismaClient = runOpsTopology.legac export const runOpsLegacyReplicaClient: RunOpsPrismaClient = runOpsTopology.legacyRunOps .replica as unknown as RunOpsPrismaClient; +// Gen-2 shard handles for the run-store boundary. Empty unless RUN_OPS_SHARDS is configured. +export const runOpsShardHandles: Array<{ + key: string; + writer: RunOpsPrismaClient; + replica: RunOpsPrismaClient; +}> = [...runOpsTopology.shards.entries()].map(([key, clients]) => ({ + key, + writer: clients.writer, + replica: clients.replica, +})); + export const runOpsSplitReadEnabled: boolean = computeRunOpsSplitReadEnabled({ newReplica: runOpsNewReplicaClient, controlPlaneWriter: prisma, @@ -924,64 +1037,64 @@ export function buildReplicaClient({ return replicaClient; } -function buildRunOpsWriterClient({ +// One factory for the run-ops writer and replica clients, backed by the dedicated RunOpsPrismaClient +// (a separately generated Prisma package). Parameterized by role and the resolved pool knobs, so a +// gen-1 new store and every gen-2 shard share this single builder. The control-plane builders +// (buildWriterClient/buildReplicaClient) are a DIFFERENT path and are untouched — this reuses only +// the shared low-level helpers (buildPrismaConnectionUrl, buildDriverAdapterPool). +function buildRunOpsClient({ url, clientType, + role, + connectionLimit, + poolTimeout, + connectTimeout, useDriverAdapter = false, }: { url: string; clientType: string; + role: "writer" | "replica"; + connectionLimit: number; + poolTimeout: number; + connectTimeout: number; useDriverAdapter?: boolean; }): RunOpsPrismaClient { - const databaseUrl = buildPrismaConnectionUrl(url, { - connectionLimit: env.DATABASE_CONNECTION_LIMIT.toString(), - poolTimeout: (env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT).toString(), - connectTimeout: ( - env.RUN_OPS_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT - ).toString(), + const isWriter = role === "writer"; + const setupLabel = isWriter ? "run-ops prisma client" : "run-ops read replica connection"; + const connectedLabel = isWriter + ? "run-ops prisma client connected" + : "run-ops read replica connected"; + + const connectionUrl = buildPrismaConnectionUrl(url, { + connectionLimit: connectionLimit.toString(), + poolTimeout: poolTimeout.toString(), + connectTimeout: connectTimeout.toString(), applicationName: env.SERVICE_NAME, }); console.log( - `🔌 setting up run-ops prisma client to ${redactUrlSecrets(databaseUrl)}${ + `🔌 setting up ${setupLabel} to ${redactUrlSecrets(connectionUrl)}${ useDriverAdapter ? " (pg driver adapter)" : "" }` ); + const log = [ + { emit: "event", level: "error" }, + { emit: "event", level: "info" }, + { emit: "event", level: "warn" }, + ...((process.env.VERBOSE_PRISMA_LOGS === "1" || + process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined + ? [{ emit: "event", level: "query" }] + : []) as { emit: "event"; level: "query" }[]), + ] as const; + const driverPool = useDriverAdapter - ? buildDriverAdapterPool( - url, - clientType, - env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, - env.DATABASE_CONNECTION_LIMIT - ) + ? buildDriverAdapterPool(url, clientType, poolTimeout, connectionLimit) : undefined; const client = driverPool - ? new RunOpsPrismaClient({ - adapter: driverPool.adapter, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }) - : new RunOpsPrismaClient({ - datasources: { db: { url: databaseUrl.href } }, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }); + ? new RunOpsPrismaClient({ adapter: driverPool.adapter, log: [...log] }) + : new RunOpsPrismaClient({ datasources: { db: { url: connectionUrl.href } }, log: [...log] }); registerDatabaseMetricsSource( driverPool @@ -999,117 +1112,26 @@ function buildRunOpsWriterClient({ client.$on("info", (log) => logger.info("RunOpsPrismaClient info", { clientType, event: log })); client.$on("warn", (log) => logger.warn("RunOpsPrismaClient warn", { clientType, event: log })); client.$on("error", (log) => - logger.error("RunOpsPrismaClient error", { clientType, event: log, ignoreError: true }) - ); - } - - client.$on("query", (log) => queryPerformanceMonitor.onQuery("writer", log)); - - const connectPromise = client.$connect(); - if (env.NODE_ENV === "test") { - connectPromise.catch((error) => { - logger.warn("Failed to eagerly connect run-ops prisma client (writer)", { error }); - }); - } - - console.log(`🔌 run-ops prisma client connected`); - - return client; -} - -function buildRunOpsReplicaClient({ - url, - clientType, - useDriverAdapter = false, -}: { - url: string; - clientType: string; - useDriverAdapter?: boolean; -}): RunOpsPrismaClient { - const replicaUrl = buildPrismaConnectionUrl(url, { - connectionLimit: ( - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT - ).toString(), - poolTimeout: ( - env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT - ).toString(), - connectTimeout: ( - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT - ).toString(), - applicationName: env.SERVICE_NAME, - }); - - console.log( - `🔌 setting up run-ops read replica connection to ${redactUrlSecrets(replicaUrl)}${ - useDriverAdapter ? " (pg driver adapter)" : "" - }` - ); - - const driverPool = useDriverAdapter - ? buildDriverAdapterPool( - url, + // The writer bridges P2002 -> 422 at the store boundary, so its infra errors are logged once + // there (ignoreError). Replica errors are not on that write path, so they log normally. + logger.error("RunOpsPrismaClient error", { clientType, - env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, - env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT - ) - : undefined; - - const client = driverPool - ? new RunOpsPrismaClient({ - adapter: driverPool.adapter, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], + event: log, + ...(isWriter ? { ignoreError: true } : {}), }) - : new RunOpsPrismaClient({ - datasources: { db: { url: replicaUrl.href } }, - log: [ - { emit: "event", level: "error" }, - { emit: "event", level: "info" }, - { emit: "event", level: "warn" }, - ...((process.env.VERBOSE_PRISMA_LOGS === "1" || - process.env.VERY_SLOW_QUERY_THRESHOLD_MS !== undefined - ? [{ emit: "event", level: "query" }] - : []) as { emit: "event"; level: "query" }[]), - ], - }); - - registerDatabaseMetricsSource( - driverPool - ? { - clientType, - usesDriverAdapter: true, - client, - pool: driverPool.pool, - poolCounters: driverPool.poolCounters, - } - : { clientType, usesDriverAdapter: false, client } - ); - - if (process.env.PRISMA_LOG_TO_STDOUT !== "1") { - client.$on("info", (log) => logger.info("RunOpsPrismaClient info", { clientType, event: log })); - client.$on("warn", (log) => logger.warn("RunOpsPrismaClient warn", { clientType, event: log })); - client.$on("error", (log) => - logger.error("RunOpsPrismaClient error", { clientType, event: log }) ); } - client.$on("query", (log) => queryPerformanceMonitor.onQuery("replica", log)); + client.$on("query", (log) => queryPerformanceMonitor.onQuery(role, log)); const connectPromise = client.$connect(); if (env.NODE_ENV === "test") { connectPromise.catch((error) => { - logger.warn("Failed to eagerly connect run-ops prisma client (replica)", { error }); + logger.warn(`Failed to eagerly connect run-ops prisma client (${role})`, { error }); }); } - console.log(`🔌 run-ops read replica connected`); + console.log(`🔌 ${connectedLabel}`); return client; } diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index c917930612..dcbb751ec1 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { MachinePresetName } from "@trigger.dev/core/v3"; import { BoolEnv } from "./utils/boolEnv"; import { isValidDatabaseUrl } from "./utils/db"; +import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server"; import { isValidRegex } from "./utils/regex"; import { isValidDuration } from "./services/realtime/duration.server"; @@ -310,6 +311,8 @@ const EnvironmentSchema = z RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER: z.string().default("0"), RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER: z.string().default("0"), RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER: z.string().default("0"), + // Gen-2 shard descriptors as a JSON array. Unset/"" -> [] (today). See runOpsShards.server.ts. + RUN_OPS_SHARDS: z.string().optional().transform(parseRunOpsShards), // Control-plane cache relax knobs. Unset -> defaults (DEFAULT_CP_CACHE_TTL_MS / _MAX_ENTRIES). CONTROL_PLANE_CACHE_TTL_MS: z.coerce.number().int().optional(), CONTROL_PLANE_CACHE_MAX_ENTRIES: z.coerce.number().int().optional(), @@ -2467,6 +2470,14 @@ const EnvironmentSchema = z }); } } + if (!validateShardListAgainstNewUrl(env.RUN_OPS_SHARDS, env.RUN_OPS_DATABASE_URL)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["RUN_OPS_SHARDS"], + message: + "RUN_OPS_SHARDS is non-empty but RUN_OPS_DATABASE_URL is unset; a shard requires the gen-1 new store", + }); + } }); export type Environment = z.infer; diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts index d88e64e1d7..a4e4a64d36 100644 --- a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts @@ -473,3 +473,40 @@ describe("computeMintShard — the global override wins the complete cutover", ( ); }); }); + +describe("routableKeys bound (the shard descriptor keys this deployment can route)", () => { + it("drops an active key that is not routable, so the hash never returns it", () => { + // "z" is in the active list but not configured as a descriptor -> only "a" is selectable. + const ids = envIds(200); + for (const id of ids) { + const shard = computeMintShard({ id }, deps({ set: ["a", "z"] }, { routableKeys: ["a"] })); + expect(shard).toBe("a"); + } + }); + + it("returns new when the active list holds only non-routable keys (fail-safe to gen-1)", () => { + expect(computeMintShard({ id: "env_1" }, deps({ set: ["z"] }, { routableKeys: ["a"] }))).toBe( + "new" + ); + }); + + it("rejects a per-org pin to a non-routable key and falls through to the hash", () => { + const shard = computeMintShard( + { id: "env_1" }, + deps({ set: ["a", "z"] }, { ...orgFlags({ runOpsMintShard: "z" }), routableKeys: ["a"] }) + ); + expect(shard).toBe("a"); + }); + + it("with no routableKeys given, behaviour is unchanged", () => { + const ids = envIds(200); + for (const id of ids) { + const withBound = computeMintShard( + { id }, + deps({ set: ["a", "b"] }, { routableKeys: ["a", "b"] }) + ); + const without = computeMintShard({ id }, deps({ set: ["a", "b"] })); + expect(withBound).toBe(without); + } + }); +}); diff --git a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts index a49a1a6a60..2855f93624 100644 --- a/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts +++ b/apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts @@ -19,6 +19,10 @@ export type MintShardDeps = { nowMs: number; graceMs: number; orgFeatureFlags: unknown; + // The shard keys this deployment can actually route (the RUN_OPS_SHARDS descriptor keys). The + // active set is bounded to these, so a stored key with no descriptor is never minted into. + // Undefined means "no bound" (today's behaviour). + routableKeys?: readonly string[]; onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void; onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void; }; @@ -94,7 +98,17 @@ function hrwSelect(environmentId: string, activeSet: string[]): string { // would leak the drain the active list performs, and throwing would fail customer triggers // whenever a pinned shard drains. export function computeMintShard(environment: { id: string }, deps: MintShardDeps): ShardKey { - const activeSet = effectiveMintShardSet(deps.resolution, deps.nowMs, deps.graceMs); + const rawActiveSet = effectiveMintShardSet(deps.resolution, deps.nowMs, deps.graceMs); + // Empty check BEFORE the bound, so an unconfigured deployment returns "new" exactly as today. + if (rawActiveSet.length === 0) { + return "new"; + } + + // Bound the active set to the keys this deployment can route. A stored key with no descriptor is + // dropped, never minted into. If nothing survives, fall back to gen-1 (fail-safe, never a throw). + const activeSet = deps.routableKeys + ? rawActiveSet.filter((key) => deps.routableKeys!.includes(key)) + : rawActiveSet; if (activeSet.length === 0) { return "new"; } @@ -148,6 +162,7 @@ export type ResolveMintShardDeps = { ttlMs: number; graceMs: number; orgFeatureFlags: unknown; + routableKeys?: readonly string[]; onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void; onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void; onReadFailed?: (error: unknown) => void; @@ -200,6 +215,7 @@ export async function resolveMintShardWith( nowMs: deps.nowMs, graceMs: deps.graceMs, orgFeatureFlags: deps.orgFeatureFlags, + routableKeys: deps.routableKeys, onPinRejected: deps.onPinRejected, onOverrideRejected: deps.onOverrideRejected, }); diff --git a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts index 542384e16f..c1c2b9ddd4 100644 --- a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts @@ -84,6 +84,8 @@ export async function resolveMintShard(environment: { ttlMs: env.RUN_OPS_MINT_FLAG_CACHE_TTL_MS, graceMs: env.RUN_OPS_MINT_FLIP_GRACE_MS, orgFeatureFlags: environment.orgFeatureFlags, + // Bound the active list to the shards this deployment can actually route. + routableKeys: env.RUN_OPS_SHARDS.map((shard) => shard.key), onPinRejected: reportPinRejected, onOverrideRejected: reportOverrideRejected, onReadFailed: (error) => diff --git a/apps/webapp/app/v3/runOpsPoolKnobs.server.ts b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts new file mode 100644 index 0000000000..0396af0f6d --- /dev/null +++ b/apps/webapp/app/v3/runOpsPoolKnobs.server.ts @@ -0,0 +1,80 @@ +import { env } from "~/env.server"; +import type { RunOpsShardKnobs } from "~/v3/runOpsShards.server"; + +// Pool configuration for one run-ops store (writer + replica). Kept separate from db.server (which +// ~156 tests mock wholesale) so a new export breaks no mock. +export type ResolvedPoolKnobs = { + writerPoolTimeout: number; + writerConnectionTimeout: number; + writerDriverAdapter: boolean; + connectionLimit: number; + replicaConnectionLimit: number; + replicaPoolTimeout: number; + replicaConnectionTimeout: number; + replicaDriverAdapter: boolean; +}; + +type Role = "new" | "legacy"; + +// PURE: overlay a gen-2 shard's descriptor knobs on a role's resolved defaults. This holds the only +// logic (per-field override), so a test drives it with literal defaults and literal overrides — +// no env import, no circular assertion against the same env expression the impl reads. +export function applyPoolKnobOverrides( + defaults: ResolvedPoolKnobs, + k?: RunOpsShardKnobs +): ResolvedPoolKnobs { + return { + writerPoolTimeout: k?.writerPoolTimeout ?? defaults.writerPoolTimeout, + writerConnectionTimeout: k?.writerConnectionTimeout ?? defaults.writerConnectionTimeout, + writerDriverAdapter: k?.writerDriverAdapter ?? defaults.writerDriverAdapter, + connectionLimit: k?.connectionLimit ?? defaults.connectionLimit, + replicaConnectionLimit: k?.replicaConnectionLimit ?? defaults.replicaConnectionLimit, + replicaPoolTimeout: k?.replicaPoolTimeout ?? defaults.replicaPoolTimeout, + replicaConnectionTimeout: k?.replicaConnectionTimeout ?? defaults.replicaConnectionTimeout, + replicaDriverAdapter: k?.replicaDriverAdapter ?? defaults.replicaDriverAdapter, + }; +} + +// The env-derived defaults for a role, reproducing today's run-ops builder expressions exactly. A +// flat mapping (no logic), verified by inspection against the former builders. Transaction +// resilience is a SEPARATE mechanism (resolveTransactionResilience) and is not here. +function poolKnobDefaults(role: Role): ResolvedPoolKnobs { + if (role === "legacy") { + return { + writerPoolTimeout: + env.RUN_OPS_LEGACY_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + writerConnectionTimeout: + env.RUN_OPS_LEGACY_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, + writerDriverAdapter: env.RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER === "1", + connectionLimit: env.DATABASE_CONNECTION_LIMIT, + replicaConnectionLimit: env.DATABASE_CONNECTION_LIMIT, + replicaPoolTimeout: + env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + replicaConnectionTimeout: + env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? + env.DATABASE_CONNECTION_TIMEOUT, + replicaDriverAdapter: env.RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + }; + } + + return { + writerPoolTimeout: env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + writerConnectionTimeout: + env.RUN_OPS_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, + writerDriverAdapter: env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1", + connectionLimit: env.DATABASE_CONNECTION_LIMIT, + replicaConnectionLimit: + env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT, + replicaPoolTimeout: env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT, + replicaConnectionTimeout: + env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT, + replicaDriverAdapter: env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1", + }; +} + +export function resolveRunOpsPoolKnobs( + role: Role, + descriptorKnobs?: RunOpsShardKnobs +): ResolvedPoolKnobs { + return applyPoolKnobOverrides(poolKnobDefaults(role), descriptorKnobs); +} diff --git a/apps/webapp/app/v3/runOpsShardTable.ts b/apps/webapp/app/v3/runOpsShardTable.ts new file mode 100644 index 0000000000..6741b3cb5b --- /dev/null +++ b/apps/webapp/app/v3/runOpsShardTable.ts @@ -0,0 +1,28 @@ +// Pure boot-table helpers. Dependency-free (no db.server, no env) so a test of these two string +// functions never constructs a Prisma client. db.server imports them for the boot log. + +// A host:port/db address, with NO username and NO query params — never a secret, and deliberately +// NOT an identity claim (two DSNs can share an address yet be different databases; that proof is the +// distinctness sentinel's, not this line's). Same tuple sameDatabaseTarget compares, kept in step. +export function runOpsAddressFingerprint(url: string): string { + try { + const u = new URL(url); + return `${u.hostname}:${u.port || "5432"}${u.pathname}`; + } catch { + return "unparseable"; + } +} + +export type RunOpsShardTableRow = { key: string; fingerprint: string; role: string }; + +// The resolved shard table for the boot log: one row per descriptor. An alias reports its role and +// carries no address (it shares the new store's pool). +export function buildRunOpsShardTable( + descriptors: Array<{ key: string; url?: string; aliasOf?: "new" }> +): RunOpsShardTableRow[] { + return descriptors.map((d) => + d.aliasOf + ? { key: d.key, fingerprint: "alias(new)", role: "alias(new)" } + : { key: d.key, fingerprint: runOpsAddressFingerprint(d.url ?? ""), role: "shard" } + ); +} diff --git a/apps/webapp/app/v3/runOpsShards.server.ts b/apps/webapp/app/v3/runOpsShards.server.ts new file mode 100644 index 0000000000..23c45efa40 --- /dev/null +++ b/apps/webapp/app/v3/runOpsShards.server.ts @@ -0,0 +1,124 @@ +import { z } from "zod"; +import { isValidShardChar } from "@trigger.dev/core/v3/isomorphic"; +import { isValidDatabaseUrl } from "~/utils/db"; + +const KnobsSchema = z + .object({ + writerPoolTimeout: z.number().int().optional(), + writerConnectionTimeout: z.number().int().optional(), + writerDriverAdapter: z.boolean().optional(), + connectionLimit: z.number().int().optional(), + replicaConnectionLimit: z.number().int().optional(), + replicaPoolTimeout: z.number().int().optional(), + replicaConnectionTimeout: z.number().int().optional(), + replicaDriverAdapter: z.boolean().optional(), + transactionMaxWaitMs: z.number().int().optional(), + transactionStartRetryEnabled: z.boolean().optional(), + transactionStartRetryMaxAttempts: z.number().int().optional(), + transactionStartRetryBackoffMinMs: z.number().int().optional(), + transactionStartRetryBackoffMaxMs: z.number().int().optional(), + transactionStartRetryBudgetPerSec: z.number().int().optional(), + transactionStartRetryBudgetBurst: z.number().int().optional(), + }) + .strict(); +export type RunOpsShardKnobs = z.infer; + +const ReplicationSchema = z.object({ + slotName: z.string().min(1), + publicationName: z.string().min(1), + originGeneration: z.number().int().min(2).max(255), +}); + +const DescriptorSchema = z + .object({ + key: z.string().refine(isValidShardChar, "shard key must be a single [a-z0-9] char"), + region: z.string().min(1), + url: z.string().refine(isValidDatabaseUrl, "url is invalid").optional(), + replicaUrl: z.string().refine(isValidDatabaseUrl, "replicaUrl is invalid").optional(), + directUrl: z.string().refine(isValidDatabaseUrl, "directUrl is invalid").optional(), + replication: ReplicationSchema.optional(), + knobs: KnobsSchema.optional(), + aliasOf: z.literal("new").optional(), + }) + .strict() + .superRefine((d, ctx) => { + const hasUrl = d.url !== undefined; + const hasAlias = d.aliasOf !== undefined; + if (hasUrl === hasAlias) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "exactly one of url or aliasOf is required", + }); + } + if (!hasAlias && d.replication === undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "replication is required unless aliasOf is set", + }); + } + }); + +export type RunOpsShardDescriptor = z.infer; + +// Boot-validated transform, in the style of parseMachinePresetCsv. Undefined and "" both mean the +// off state and resolve to []. The undefined guard is load-bearing: an unguarded JSON.parse would +// kill every single-DB boot, which never sets this variable. +export function parseRunOpsShards( + raw: string | undefined, + ctx: z.RefinementCtx +): RunOpsShardDescriptor[] { + if (raw === undefined || raw.trim() === "") return []; + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "RUN_OPS_SHARDS is not valid JSON" }); + return z.NEVER; + } + + const result = z.array(DescriptorSchema).safeParse(parsed); + if (!result.success) { + for (const issue of result.error.issues) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `RUN_OPS_SHARDS[${issue.path.join(".")}]: ${issue.message}`, + }); + } + return z.NEVER; + } + + const keys = new Set(); + const gens = new Set(); + for (const d of result.data) { + if (keys.has(d.key)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `RUN_OPS_SHARDS: duplicate key ${d.key}`, + }); + return z.NEVER; + } + keys.add(d.key); + if (d.replication) { + if (gens.has(d.replication.originGeneration)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `RUN_OPS_SHARDS: duplicate originGeneration ${d.replication.originGeneration}`, + }); + return z.NEVER; + } + gens.add(d.replication.originGeneration); + } + } + + return result.data; +} + +// A non-empty shard list requires the gen-1 new store, because gen-1 v1 ids resolve to "new" +// forever (append-only). Pure so the boot refinement and its test share one rule. +export function validateShardListAgainstNewUrl( + shards: RunOpsShardDescriptor[], + newUrl: string | undefined +): boolean { + return shards.length === 0 || !!newUrl; +} diff --git a/apps/webapp/app/v3/runStore.server.ts b/apps/webapp/app/v3/runStore.server.ts index 9ccf84b511..3fd0fcfa3d 100644 --- a/apps/webapp/app/v3/runStore.server.ts +++ b/apps/webapp/app/v3/runStore.server.ts @@ -1,5 +1,10 @@ import { PostgresRunStore, RoutingRunStore, type RunStore } from "@internal/run-store"; -import { ownerEngine, type Residency } from "@trigger.dev/core/v3/isomorphic"; +import { + ownerEngine, + resolveShard, + type Residency, + type ShardKey, +} from "@trigger.dev/core/v3/isomorphic"; import type { PrismaClient, PrismaReplicaClient } from "@trigger.dev/database"; import type { RunOpsPrismaClient } from "@internal/run-ops-database"; import { @@ -9,6 +14,7 @@ import { runOpsLegacyReplica, runOpsNewPrismaClient, runOpsNewReplicaClient, + runOpsShardHandles, } from "~/db.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; @@ -31,6 +37,16 @@ type BuildRunStoreDeps = { singleReplica: PrismaReplicaClient; /** Residency classifier; defaults to ownerEngine inside RoutingRunStore. */ classify?: (id: string) => Residency; + /** Gen-2 shard handles. When non-empty, buildRunStore produces N dedicated stores + the keyed + * router (fromShards). Empty/absent keeps today's two-store compat router. */ + shards?: Array<{ + key: ShardKey; + writer: RunOpsPrismaClient; + replica: RunOpsPrismaClient; + resilience?: TransactionResilienceConfig; + }>; + /** Shard-key resolver for the fromShards path; defaults to resolveShard. */ + resolveShardKey?: (id: string) => ShardKey; /** Per-pool transaction-resilience configs threaded into the store(s) this builds (IoC). */ singleResilience?: TransactionResilienceConfig; newResilience?: TransactionResilienceConfig; @@ -79,10 +95,49 @@ export function buildRunStore(deps: BuildRunStoreDeps): RunStore { transactionStartRetry: deps.legacyResilience?.startRetry, }); - return new RoutingRunStore({ - new: newStore, - legacy: legacyStore, - classify: deps.classify ?? ownerEngine, + // No gen-2 shards: today's two-store compat router, byte-identical. + if (!deps.shards || deps.shards.length === 0) { + return new RoutingRunStore({ + new: newStore, + legacy: legacyStore, + classify: deps.classify ?? ownerEngine, + }); + } + + // Gen-2 shards: one dedicated store per descriptor, then the keyed N-way router. Every shard is a + // schemaVariant:"dedicated" instance, exactly like the gen-1 new store. + const shardStores = deps.shards.map((shard) => ({ + key: shard.key, + store: new PostgresRunStore({ + prisma: shard.writer, + readOnlyPrisma: shard.replica, + schemaVariant: "dedicated", + maxWait: shard.resilience?.maxWait, + transactionStartRetry: shard.resilience?.startRetry, + }), + })); + + const shardKeys = shardStores.map((s) => s.key); + const shardMap = new Map([ + ["legacy", legacyStore], + ["new", newStore], + ...shardStores.map(({ key, store }) => [key, store] as const), + ]); + + // Ascending authority for a merge: legacy -> new -> shards in configured order. The router + // requires probeOrder to be the exact reverse (see the class invariant in runOpsStore.ts), so a + // duplicate id resolves the same way on the merge path and the probe path. + const precedence: ShardKey[] = ["legacy", "new", ...shardKeys]; + const probeOrder = [...precedence].reverse(); + + return RoutingRunStore.fromShards({ + shards: shardMap, + precedence, + probeOrder, + idlessRouteShard: "new", + idlessWaitpointShard: "legacy", + resolveShardKey: deps.resolveShardKey ?? resolveShard, + classify: deps.classify, }); } @@ -110,6 +165,8 @@ function tryResolveRunOpsHandles() { newReplica: runOpsNewReplicaClient, legacyWriter: runOpsLegacyPrisma, legacyReplica: runOpsLegacyReplica, + // Absent under a minimal db.server mock; default to no shards so the compat router is built. + shardHandles: runOpsShardHandles ?? [], }; } catch { return null; @@ -127,9 +184,16 @@ export const runStore: RunStore = singleton("RunStore", () => { singleResilience: resilienceForClient(prisma), }); } + const { shardHandles, ...storeHandles } = handles; return buildRunStore({ splitEnabled: true, - ...handles, + ...storeHandles, + shards: shardHandles.map((shard) => ({ + key: shard.key, + writer: shard.writer, + replica: shard.replica, + resilience: resilienceForClient(shard.writer), + })), singleWriter: prisma, singleReplica: $replica, singleResilience: resilienceForClient(prisma), diff --git a/apps/webapp/app/v3/transactionResilience.server.ts b/apps/webapp/app/v3/transactionResilience.server.ts index ae5678c987..eabde6691d 100644 --- a/apps/webapp/app/v3/transactionResilience.server.ts +++ b/apps/webapp/app/v3/transactionResilience.server.ts @@ -17,8 +17,11 @@ export type TransactionResilienceConfig = { startRetry: TransactionStartRetryConfig; }; -function resolveTransactionResilience( - pool: "control-plane" | "run-ops" | "run-ops-legacy", +// Exported so the topology singleton can build a per-shard config (each call creates its OWN +// TokenBucketRetryBudget, so one shard's retry storm cannot drain another's). `pool` is a free +// string — it only labels a log line, never keys any behaviour. +export function resolveTransactionResilience( + pool: string, overrides: { maxWaitMs?: number; enabled?: boolean; @@ -64,6 +67,44 @@ export const runOpsTransactionResilience = resolveTransactionResilience("run-ops budgetBurst: env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BUDGET_BURST, }); +// A gen-2 shard's resilience. Defaults to the RUN_OPS_DATABASE_TRANSACTION_* values (so a shard with +// no overrides matches the gen-1 new store), then applies the descriptor's per-shard overrides. Each +// call builds its OWN budget, so a storm on one shard cannot drain another's. +export function resolveShardResilience( + key: string, + overrides?: { + transactionMaxWaitMs?: number; + transactionStartRetryEnabled?: boolean; + transactionStartRetryMaxAttempts?: number; + transactionStartRetryBackoffMinMs?: number; + transactionStartRetryBackoffMaxMs?: number; + transactionStartRetryBudgetPerSec?: number; + transactionStartRetryBudgetBurst?: number; + } +): TransactionResilienceConfig { + return resolveTransactionResilience(`run-ops-shard-${key}`, { + maxWaitMs: overrides?.transactionMaxWaitMs ?? env.RUN_OPS_DATABASE_TRANSACTION_MAX_WAIT_MS, + enabled: + overrides?.transactionStartRetryEnabled ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_ENABLED, + maxAttempts: + overrides?.transactionStartRetryMaxAttempts ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_MAX_ATTEMPTS, + backoffMinMs: + overrides?.transactionStartRetryBackoffMinMs ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BACKOFF_MIN_MS, + backoffMaxMs: + overrides?.transactionStartRetryBackoffMaxMs ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BACKOFF_MAX_MS, + budgetPerSec: + overrides?.transactionStartRetryBudgetPerSec ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BUDGET_PER_SEC, + budgetBurst: + overrides?.transactionStartRetryBudgetBurst ?? + env.RUN_OPS_DATABASE_TRANSACTION_START_RETRY_BUDGET_BURST, + }); +} + export const runOpsLegacyTransactionResilience = resolveTransactionResilience("run-ops-legacy", { maxWaitMs: env.RUN_OPS_LEGACY_DATABASE_TRANSACTION_MAX_WAIT_MS, enabled: env.RUN_OPS_LEGACY_DATABASE_TRANSACTION_START_RETRY_ENABLED, diff --git a/apps/webapp/test/runOpsDbTopology.test.ts b/apps/webapp/test/runOpsDbTopology.test.ts index 8890fdbb66..f2bcc0bf5e 100644 --- a/apps/webapp/test/runOpsDbTopology.test.ts +++ b/apps/webapp/test/runOpsDbTopology.test.ts @@ -142,6 +142,75 @@ describe("selectRunOpsTopology (pure)", () => { expect(topo.legacyRunOps.replica).toBe(legacyWriter); expect(buildLegacyReplica).not.toHaveBeenCalled(); }); + + const baseSplit = { + splitEnabled: true, + legacyUrl: "postgres://legacy", + newUrl: "postgres://new", + }; + const baseBuilders = () => ({ + controlPlane: cp, + buildNewWriter: vi.fn().mockReturnValue({ tag: "nw" } as any), + buildNewReplica: vi.fn().mockReturnValue({ tag: "nr" } as any), + buildLegacyWriter: vi.fn().mockReturnValue({ tag: "lw" } as any), + buildLegacyReplica: vi.fn().mockReturnValue({ tag: "lr" } as any), + }); + + it("no descriptors: the shards map is empty", () => { + const topo = selectRunOpsTopology(baseSplit, baseBuilders()); + expect(topo.shards.size).toBe(0); + }); + + it("two descriptors: two shard client pairs, each built once", () => { + const buildShardWriter = vi.fn((s: any) => ({ tag: `w:${s.key}` }) as any); + const buildShardReplica = vi.fn((s: any) => ({ tag: `r:${s.key}` }) as any); + const topo = selectRunOpsTopology( + { + ...baseSplit, + shards: [ + { key: "a", url: "postgres://a", replicaUrl: "postgres://a-r" }, + { key: "b", url: "postgres://b" }, + ], + }, + { ...baseBuilders(), buildShardWriter, buildShardReplica } + ); + expect(topo.shards.size).toBe(2); + expect(topo.shards.get("a")!.writer).toEqual({ tag: "w:a" }); + // b has no replicaUrl, so its replica falls back to its writer (buildShardReplica not called for b). + expect(topo.shards.get("b")!.replica).toEqual({ tag: "w:b" }); + expect(buildShardWriter).toHaveBeenCalledTimes(2); + expect(buildShardReplica).toHaveBeenCalledTimes(1); + }); + + it("an alias descriptor reuses newRunOps by reference and calls no shard builder", () => { + const buildShardWriter = vi.fn(); + const buildShardReplica = vi.fn(); + const topo = selectRunOpsTopology( + { ...baseSplit, shards: [{ key: "a", aliasOf: "new" }] }, + { ...baseBuilders(), buildShardWriter, buildShardReplica } + ); + expect(topo.shards.get("a")).toBe(topo.newRunOps); + expect(buildShardWriter).not.toHaveBeenCalled(); + expect(buildShardReplica).not.toHaveBeenCalled(); + }); + + it("throws when a non-aliased shard has no url (guards the shard.url non-null assertion)", () => { + expect(() => + selectRunOpsTopology( + { ...baseSplit, shards: [{ key: "a" }] }, + { ...baseBuilders(), buildShardWriter: vi.fn(), buildShardReplica: vi.fn() } + ) + ).toThrow(/shard "a" needs a url/); + }); + + it("throws when a non-aliased shard is configured but the shard builders are absent", () => { + expect(() => + selectRunOpsTopology( + { ...baseSplit, shards: [{ key: "a", url: "postgres://a" }] }, + baseBuilders() + ) + ).toThrow(/shard "a" needs a url and shard builders/); + }); }); describe("sameDatabaseTarget", () => { diff --git a/apps/webapp/test/runOpsPoolKnobs.test.ts b/apps/webapp/test/runOpsPoolKnobs.test.ts new file mode 100644 index 0000000000..38353682cb --- /dev/null +++ b/apps/webapp/test/runOpsPoolKnobs.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { applyPoolKnobOverrides, type ResolvedPoolKnobs } from "~/v3/runOpsPoolKnobs.server"; + +// Literal defaults, so the assertions lock the override logic against fixed values rather than +// against the same env expression the implementation reads. No env import (webapp test rule). +const DEFAULTS: ResolvedPoolKnobs = { + writerPoolTimeout: 10, + writerConnectionTimeout: 20, + writerDriverAdapter: false, + connectionLimit: 30, + replicaConnectionLimit: 40, + replicaPoolTimeout: 50, + replicaConnectionTimeout: 60, + replicaDriverAdapter: false, +}; + +describe("applyPoolKnobOverrides", () => { + it("returns the defaults verbatim when no descriptor knobs are given", () => { + expect(applyPoolKnobOverrides(DEFAULTS)).toEqual(DEFAULTS); + expect(applyPoolKnobOverrides(DEFAULTS, {})).toEqual(DEFAULTS); + }); + + it("overrides only the fields the descriptor sets", () => { + const result = applyPoolKnobOverrides(DEFAULTS, { + connectionLimit: 999, + writerDriverAdapter: true, + replicaPoolTimeout: 555, + }); + expect(result.connectionLimit).toBe(999); + expect(result.writerDriverAdapter).toBe(true); + expect(result.replicaPoolTimeout).toBe(555); + // Untouched fields keep the defaults. + expect(result.writerPoolTimeout).toBe(10); + expect(result.replicaConnectionLimit).toBe(40); + expect(result.replicaDriverAdapter).toBe(false); + }); + + it("does not read the transaction knobs off the descriptor", () => { + const result = applyPoolKnobOverrides(DEFAULTS, { transactionMaxWaitMs: 1234 }); + expect(result).toEqual(DEFAULTS); + }); +}); diff --git a/apps/webapp/test/runOpsShardBootTable.test.ts b/apps/webapp/test/runOpsShardBootTable.test.ts new file mode 100644 index 0000000000..3031608d70 --- /dev/null +++ b/apps/webapp/test/runOpsShardBootTable.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { runOpsAddressFingerprint, buildRunOpsShardTable } from "~/v3/runOpsShardTable"; + +describe("runOpsAddressFingerprint", () => { + it("returns host:port/db with no username or query params", () => { + const fp = runOpsAddressFingerprint( + "postgres://user:pw@host.example:5433/mydb?schema=public&pool_timeout=20" + ); + expect(fp).toBe("host.example:5433/mydb"); + expect(fp).not.toContain("user"); + expect(fp).not.toContain("pool_timeout"); + }); + it("defaults the port to 5432", () => { + expect(runOpsAddressFingerprint("postgres://h/db")).toBe("h:5432/db"); + }); + it("returns a marker on unparseable input rather than throwing", () => { + expect(runOpsAddressFingerprint("not a url")).toBe("unparseable"); + }); +}); + +describe("buildRunOpsShardTable", () => { + it("one row per descriptor, with key, fingerprint, and role", () => { + const rows = buildRunOpsShardTable([ + { key: "a", url: "postgres://user:pw@h/adb?schema=public" }, + { key: "b", aliasOf: "new" }, + ]); + expect(rows).toHaveLength(2); + expect(rows[0]).toEqual({ key: "a", fingerprint: "h:5432/adb", role: "shard" }); + expect(rows[1]).toEqual({ key: "b", fingerprint: "alias(new)", role: "alias(new)" }); + }); + it("is empty for an empty descriptor list", () => { + expect(buildRunOpsShardTable([])).toEqual([]); + }); +}); diff --git a/apps/webapp/test/runOpsShards.test.ts b/apps/webapp/test/runOpsShards.test.ts new file mode 100644 index 0000000000..fef7e925e7 --- /dev/null +++ b/apps/webapp/test/runOpsShards.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; +import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server"; + +function run(raw: string | undefined) { + const schema = z.string().optional().transform(parseRunOpsShards); + return schema.safeParse(raw); +} + +const valid = { + key: "a", + region: "us-east-1", + url: "postgres://h/db", + replication: { slotName: "s", publicationName: "p", originGeneration: 2 }, +}; + +describe("parseRunOpsShards", () => { + it("returns [] for undefined", () => { + const r = run(undefined); + expect(r.success && r.data).toEqual([]); + }); + it("returns [] for an empty array literal", () => { + const r = run("[]"); + expect(r.success && r.data).toEqual([]); + }); + it("parses a valid single descriptor", () => { + const r = run(JSON.stringify([valid])); + expect(r.success).toBe(true); + if (r.success) expect(r.data[0].key).toBe("a"); + }); + it("fails on malformed JSON", () => { + expect(run("{not json").success).toBe(false); + }); + it("fails on a multi-char key", () => { + expect(run(JSON.stringify([{ ...valid, key: "ab" }])).success).toBe(false); + }); + it("fails on duplicate keys", () => { + const b = { + ...valid, + replication: { slotName: "s2", publicationName: "p2", originGeneration: 3 }, + }; + expect(run(JSON.stringify([valid, b])).success).toBe(false); + }); + it("fails on duplicate origin generations", () => { + const b = { ...valid, key: "b", url: "postgres://h/b" }; + expect(run(JSON.stringify([valid, b])).success).toBe(false); + }); + it("fails when both url and aliasOf are set", () => { + expect( + run(JSON.stringify([{ key: "a", region: "x", url: "postgres://h/db", aliasOf: "new" }])) + .success + ).toBe(false); + }); + it("accepts aliasOf without url or replication", () => { + expect(run(JSON.stringify([{ key: "a", region: "x", aliasOf: "new" }])).success).toBe(true); + }); + it("fails on an origin generation below 2 or above 255", () => { + const mk = (g: number) => + run( + JSON.stringify([ + { ...valid, replication: { slotName: "s", publicationName: "p", originGeneration: g } }, + ]) + ); + expect(mk(1).success).toBe(false); + expect(mk(256).success).toBe(false); + }); + it("fails when a non-aliased descriptor omits replication", () => { + expect(run(JSON.stringify([{ key: "a", region: "x", url: "postgres://h/db" }])).success).toBe( + false + ); + }); +}); + +describe("validateShardListAgainstNewUrl", () => { + it("passes when the list is empty and no new url", () => { + expect(validateShardListAgainstNewUrl([], undefined)).toBe(true); + }); + it("passes when the list is non-empty and new url is set", () => { + expect(validateShardListAgainstNewUrl([valid as never], "postgres://h/new")).toBe(true); + }); + it("fails when the list is non-empty and new url is unset", () => { + expect(validateShardListAgainstNewUrl([valid as never], undefined)).toBe(false); + }); +}); diff --git a/apps/webapp/test/runStoreShardWiring.test.ts b/apps/webapp/test/runStoreShardWiring.test.ts new file mode 100644 index 0000000000..728f8ee562 --- /dev/null +++ b/apps/webapp/test/runStoreShardWiring.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { RoutingRunStore } from "@internal/run-store"; +import { buildRunStore } from "~/v3/runStore.server"; + +// Construction-only: buildRunStore wraps clients but never connects, so stub handles suffice. This +// asserts the wiring shape (compat router vs N-way router), not query behaviour. +const stub = () => ({}) as any; + +const baseSplit = { + splitEnabled: true as const, + newWriter: stub(), + newReplica: stub(), + legacyWriter: stub(), + legacyReplica: stub(), + singleWriter: stub(), + singleReplica: stub(), +}; + +describe("buildRunStore shard wiring", () => { + it("split ON with no shards builds the two-store compat router", () => { + const store = buildRunStore(baseSplit); + expect(store).toBeInstanceOf(RoutingRunStore); + }); + + it("split ON with two shard descriptors builds the N-way router", () => { + const store = buildRunStore({ + ...baseSplit, + shards: [ + { key: "a", writer: stub(), replica: stub() }, + { key: "b", writer: stub(), replica: stub() }, + ], + }); + expect(store).toBeInstanceOf(RoutingRunStore); + }); + + it("split OFF builds the single-store passthrough (not a router)", () => { + const store = buildRunStore({ + splitEnabled: false, + singleWriter: stub(), + singleReplica: stub(), + }); + expect(store).not.toBeInstanceOf(RoutingRunStore); + }); +}); diff --git a/apps/webapp/test/transactionResilience.test.ts b/apps/webapp/test/transactionResilience.test.ts new file mode 100644 index 0000000000..a033f3be98 --- /dev/null +++ b/apps/webapp/test/transactionResilience.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vitest"; +import { resolveTransactionResilience } from "~/v3/transactionResilience.server"; + +describe("resolveTransactionResilience per-shard", () => { + it("builds a distinct budget per call, so one shard's storm cannot drain another's", () => { + const a = resolveTransactionResilience("run-ops-shard-a", {}); + const b = resolveTransactionResilience("run-ops-shard-b", {}); + expect(a.startRetry.budget).not.toBe(b.startRetry.budget); + }); + + it("accepts an arbitrary pool label and honours a maxWait override", () => { + expect(() => + resolveTransactionResilience("run-ops-shard-z", { maxWaitMs: 1234 }) + ).not.toThrow(); + expect(resolveTransactionResilience("run-ops-shard-z", { maxWaitMs: 1234 }).maxWait).toBe(1234); + }); +}); diff --git a/internal-packages/run-store/src/runOpsStore.fromShards.test.ts b/internal-packages/run-store/src/runOpsStore.fromShards.test.ts new file mode 100644 index 0000000000..e9f55332ad --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.fromShards.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; +import { + generateRunOpsId, + generateRunOpsIdV2, + resolveShard, + type ShardKey, +} from "@trigger.dev/core/v3/isomorphic"; +import { RoutingRunStore, UnknownShardKey } from "./runOpsStore.js"; +import type { ReadClient, RunStore } from "./types.js"; + +// Pure routing unit test for the N-way fromShards factory. Each shard is a fake RunStore whose +// findRun records which slot answered, so the assertions are purely about WHICH store the router +// selects. No database. +type FakeStore = RunStore & { slot: ShardKey }; + +function fakeStore(slot: ShardKey): FakeStore { + const store: Partial = { + slot, + primaryReadClient: { __primary: slot } as unknown as ReadClient, + findRun: ((_where: unknown, _argsOrClient?: unknown, _client?: unknown) => + Promise.resolve({ slot } as never)) as FakeStore["findRun"], + }; + return store as FakeStore; +} + +function build(shardKeys: ShardKey[]) { + const shards = new Map(); + shards.set("legacy", fakeStore("legacy")); + shards.set("new", fakeStore("new")); + for (const k of shardKeys) shards.set(k, fakeStore(k)); + return RoutingRunStore.fromShards({ + shards, + probeOrder: ["new", ...shardKeys, "legacy"], + precedence: ["legacy", "new", ...shardKeys], + idlessRouteShard: "new", + idlessWaitpointShard: "legacy", + resolveShardKey: resolveShard, + }); +} + +describe("RoutingRunStore.fromShards", () => { + it("routes a gen-2 id to its own shard, not to new", async () => { + const store = build(["a"]); + const found = await store.findRun({ friendlyId: generateRunOpsIdV2("a") }); + expect(found).toMatchObject({ slot: "a" }); + }); + + it("routes a gen-1 v1 id to new", async () => { + const store = build(["a"]); + const found = await store.findRun({ friendlyId: generateRunOpsId() }); + expect(found).toMatchObject({ slot: "new" }); + }); + + it("routes a cuid id to legacy", async () => { + const store = build(["a"]); + const found = await store.findRun({ friendlyId: "clabc123def456ghi789jkl01" }); + expect(found).toMatchObject({ slot: "legacy" }); + }); + + it("raises UnknownShardKey for an unconfigured shard and does not fall back", () => { + const store = build(["a"]); // "b" is not configured + // The route resolves synchronously, so the throw is synchronous (before the promise is built). + expect(() => store.findRun({ friendlyId: generateRunOpsIdV2("b") })).toThrow(UnknownShardKey); + }); +}); diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index 27bd78866d..8d1c7d5aba 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -40,6 +40,33 @@ import { boundedIn } from "@trigger.dev/database"; const NEW_SHARD: ShardKey = "new"; const LEGACY_SHARD: ShardKey = "legacy"; +/** + * Raised when an id resolves to a shard key that is not configured. NEVER falls back to another + * store — a misconfiguration must fail loud, not misroute silently. Alarmed by ops. + */ +export class UnknownShardKey extends Error { + readonly key: string; + readonly configuredKeys: readonly string[]; + constructor(key: string, configuredKeys: readonly string[]) { + super( + `Unknown run-ops shard key ${JSON.stringify(key)}; configured: [${configuredKeys.join(", ")}]` + ); + this.name = "UnknownShardKey"; + this.key = key; + this.configuredKeys = configuredKeys; + } +} + +type ShardTopology = { + shards: ReadonlyMap; + probeOrder: readonly ShardKey[]; + precedence: readonly ShardKey[]; + idlessRouteShard: ShardKey; + idlessWaitpointShard: ShardKey; + resolveShardKey: (id: string) => ShardKey; + classify?: (id: string) => Residency; +}; + /** * Run-ops routing substrate for the TaskRun-core method group. Implements {@link RunStore} over a * map from shard key to store, selecting one by the residency classifier (`ownerEngine`: run-ops @@ -58,18 +85,24 @@ const LEGACY_SHARD: ShardKey = "legacy"; * each other, so swapping them changes behaviour. */ export class RoutingRunStore implements RunStore { - readonly #shards: ReadonlyMap; + // Not readonly: the compat constructor sets gen-1 defaults, and fromShards() overwrites these + // once (before the instance escapes) via #applyShardTopology. + #shards: ReadonlyMap; // Sequential probe for a lookup with no routable id. The first non-null result wins, and the LAST // entry owns the canonical not-found throw. - readonly #probeOrder: readonly ShardKey[]; + #probeOrder: readonly ShardKey[]; // Ascending authority for a merge. The last write wins, so the highest-authority shard wins a // duplicate id. Every merge in this class MUST use this order. - readonly #precedence: readonly ShardKey[]; + #precedence: readonly ShardKey[]; // The two id-less defaults. They differ by role on purpose: a route with no id lands on the // steady-state home, a waitpoint read with no id lands on the legacy store. - readonly #idlessRouteShard: ShardKey; - readonly #idlessWaitpointShard: ShardKey; + #idlessRouteShard: ShardKey; + #idlessWaitpointShard: ShardKey; readonly #classify: (id: string) => Residency; + // The shard that owns an id. Compat: binary over #classify. fromShards: resolveShard, which names + // a gen-2 id's own shard. NEVER throws (resolveShard is total) — an unconfigured key is caught at + // #shardStore, so #routeKeyOrDefault's catch cannot swallow it into a silent legacy read. + #resolveShardKey: (id: string) => ShardKey; // Compat constructor: the two gen-1 stores, keyed by their reserved shard keys. The options type // MUST stay closed — a union arm loosens the excess-property check and retires the @@ -84,6 +117,28 @@ export class RoutingRunStore implements RunStore { this.#idlessRouteShard = NEW_SHARD; this.#idlessWaitpointShard = LEGACY_SHARD; this.#classify = options.classify ?? ownerEngine; + this.#resolveShardKey = (id) => (this.#classify(id) === "NEW" ? NEW_SHARD : LEGACY_SHARD); + } + + // The N-way factory. Builds via the compat constructor (so the closed options type and its + // test-corpus lock are untouched), then installs the shard topology over the gen-1 defaults. + static fromShards(topology: ShardTopology): RoutingRunStore { + const store = new RoutingRunStore({ + new: topology.shards.get(NEW_SHARD)!, + legacy: topology.shards.get(LEGACY_SHARD)!, + classify: topology.classify, + }); + store.#applyShardTopology(topology); + return store; + } + + #applyShardTopology(topology: ShardTopology): void { + this.#shards = topology.shards; + this.#probeOrder = topology.probeOrder; + this.#precedence = topology.precedence; + this.#idlessRouteShard = topology.idlessRouteShard; + this.#idlessWaitpointShard = topology.idlessWaitpointShard; + this.#resolveShardKey = topology.resolveShardKey; } // A routing store spans two databases and has no single primary — routed reads resolve the @@ -108,14 +163,17 @@ export class RoutingRunStore implements RunStore { #shardStore(key: ShardKey): RunStore { const store = this.#shards.get(key); if (store === undefined) { - throw new Error(`RoutingRunStore: no store is configured for shard key "${key}"`); + // The ONLY place an unconfigured key fails. Keep it here, never in #resolveShardKey: + // #routeKeyOrDefault catches resolver throws and downgrades to legacy, which would turn a + // misconfiguration into a silent legacy read. This throw is outside that catch. + throw new UnknownShardKey(key, [...this.#shards.keys()]); } return store; } - // The shard that owns an existing id. Throws only when an injected classifier throws. + // The shard that owns an existing id. Delegates to the installed resolver (see #resolveShardKey). #shardKeyOf(id: string): ShardKey { - return this.#classify(id) === "NEW" ? NEW_SHARD : LEGACY_SHARD; + return this.#resolveShardKey(id); } // An unclassifiable id is treated as LEGACY (probe the control-plane DB rather than drop a diff --git a/packages/core/src/v3/isomorphic/friendlyId.test.ts b/packages/core/src/v3/isomorphic/friendlyId.test.ts index b5ea7a5197..8d818ae60b 100644 --- a/packages/core/src/v3/isomorphic/friendlyId.test.ts +++ b/packages/core/src/v3/isomorphic/friendlyId.test.ts @@ -20,6 +20,7 @@ import { generateRunOpsId, generateRunOpsIdV2, generateWaitpointId, + isValidShardChar, parseRunId, parseRunOpsIdBody, parseRunOpsIdV2Body, @@ -418,6 +419,21 @@ describe("parseRunId — v2 arm", () => { }); }); +describe("isValidShardChar", () => { + it("accepts a single [a-z0-9] char", () => { + expect(isValidShardChar("a")).toBe(true); + expect(isValidShardChar("0")).toBe(true); + expect(isValidShardChar("w")).toBe(true); + }); + it("rejects multi-char, empty, uppercase, and punctuation", () => { + expect(isValidShardChar("")).toBe(false); + expect(isValidShardChar("ab")).toBe(false); + expect(isValidShardChar("A")).toBe(false); + expect(isValidShardChar("-")).toBe(false); + expect(isValidShardChar("legacy")).toBe(false); + }); +}); + describe("waitpoint ids: run-ops format with version char w", () => { it("mints a 26-char body per type, with the type char at index 24 and version w at 25", () => { const cases: Array<[WaitpointIdType, string]> = [ diff --git a/packages/core/src/v3/isomorphic/friendlyId.ts b/packages/core/src/v3/isomorphic/friendlyId.ts index 2f436b93a3..416660ce44 100644 --- a/packages/core/src/v3/isomorphic/friendlyId.ts +++ b/packages/core/src/v3/isomorphic/friendlyId.ts @@ -40,6 +40,11 @@ export const DEFAULT_REGION_CHAR = "0"; const REGION_CHAR_PATTERN = /^[a-z0-9]$/; // Same slot, same range: the gen-2 shard key is a region char's positional twin. const SHARD_CHAR_PATTERN = REGION_CHAR_PATTERN; +/** True iff `value` is a single valid gen-2 shard char. The descriptor validator and + * `resolveShard` share this so a configured key and a decoded key cannot drift. */ +export function isValidShardChar(value: string): boolean { + return SHARD_CHAR_PATTERN.test(value); +} /** One lowercase [a-z0-9] char per supported region, at RUN_OPS_ID_REGION_INDEX. */ export const REGION_CODES: Readonly> = { "us-east-1": "e",