diff --git a/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts b/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts new file mode 100644 index 00000000000..c74aa86db92 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts @@ -0,0 +1,75 @@ +// Builds the snapshot-store decorator over a real PostgresRunStore, for injection through the +// engine's `store` option — the seam runStoreInjectability.test.ts already proves. +// +// The point of injecting it is that the engine suites keep their own assertions: the same flows, +// the same expectations, a different store underneath. +import { + PostgresRunStore, + RedisSnapshotStore, + TaskRunExecutionSnapshotStore, + type SnapshotFaultInjector, + type SnapshotRepairEnqueuer, + type SnapshotStoreMode, +} from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "@internal/redis"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +export type DecoratedStoreHarness = { + store: TaskRunExecutionSnapshotStore; + redis: RedisSnapshotStore; + /** Every read the decorator served, and which store answered it. */ + reads: { method: string; source: "redis" | "postgres" }[]; + /** Every append outcome, keyed by the write site that produced it. */ + writes: { site: string; outcome: string }[]; + /** Runs handed to the repair job because their append was lost. */ + repairs: { runId: string; snapshotId: string; executionStatus: string }[]; + quit(): Promise; +}; + +export function buildDecoratedStore(opts: { + prisma: PrismaClient; + redisOptions: RedisOptions; + mode: SnapshotStoreMode; + readPercent?: number; + faults?: SnapshotFaultInjector; + onAppendFailure?: SnapshotRepairEnqueuer; +}): DecoratedStoreHarness { + const redis = new RedisSnapshotStore({ + redisOptions: opts.redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + }); + + const reads: DecoratedStoreHarness["reads"] = []; + const writes: DecoratedStoreHarness["writes"] = []; + const repairs: DecoratedStoreHarness["repairs"] = []; + + const store = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma: opts.prisma as never, readOnlyPrisma: opts.prisma as never }), + { + store: redis, + mode: opts.mode, + readPercent: opts.readPercent ?? 100, + ...(opts.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + await opts.onAppendFailure?.(args); + }, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { + store, + redis, + reads, + writes, + repairs, + quit: () => redis.quit(), + }; +} diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts new file mode 100644 index 00000000000..ea3447acc56 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts @@ -0,0 +1,402 @@ +// The correctness spine: kill the process at each write boundary and prove the run still converges. +// +// The write protocol's whole claim is that whatever a crash leaves behind is a state the existing +// stall-and-repair machinery heals. That claim is not checkable by reading the code, so each test +// here injects a fault at one named boundary and then asserts three things: the run converges, it +// does not hang, and it burns at most one attempt number per crash. +// +// The bound is PER CRASH, not a flat one. The plan records that TLC refuted a flat bound of one in +// seven states, and that the property which holds is pgAttempt - maxLoggedAttempt <= crashCount. +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { generateInternalId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { + InjectedSnapshotFault, + type SnapshotFaultBoundary, + type SnapshotFaultInjector, +} from "@internal/run-store"; +import { setTimeout } from "timers/promises"; +import { RunEngine } from "../index.js"; +import { buildDecoratedStore, type DecoratedStoreHarness } from "./helpers/decoratedStore.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +/** + * A local stand-in for the shared fault harness being built alongside this ticket. Its surface is + * the agreed one — arm, disarm, hook, fired — so swapping the import in costs no test-body change. + * + * `fired` is the guard against a silent pass. A boundary can be armed and never reached, in which + * case the test would go green having proved nothing, so every test asserts its boundary fired. + */ +function createFaultInjector(opts: { error: (boundary: SnapshotFaultBoundary) => Error }) { + const armed = new Map(); + const counts = new Map(); + + return { + arm(boundary: SnapshotFaultBoundary, opts?: { times?: number; runId?: string }) { + armed.set(boundary, { times: opts?.times ?? 1, ...(opts?.runId && { runId: opts.runId }) }); + }, + disarm(boundary: SnapshotFaultBoundary) { + armed.delete(boundary); + }, + fired(boundary: SnapshotFaultBoundary): number { + return counts.get(boundary) ?? 0; + }, + hook: ((boundary, context) => { + const entry = armed.get(boundary); + if (!entry) return; + if (entry.runId && context.runId !== entry.runId) return; + + counts.set(boundary, (counts.get(boundary) ?? 0) + 1); + entry.times -= 1; + if (entry.times <= 0) armed.delete(boundary); + + throw opts.error(boundary); + }) satisfies SnapshotFaultInjector, + }; +} + +function engineOptions( + prisma: any, + redisOptions: any, + harness: DecoratedStoreHarness, + heartbeatMs: number +) { + return { + prisma, + store: harness.store, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + retryOptions: { maxTimeoutInMs: 50 }, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + heartbeatTimeoutsMs: { PENDING_EXECUTING: heartbeatMs }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerArgs = (taskIdentifier: string, environment: any) => ({ + number: 1, + friendlyId: RunId.generate().friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `t_${generateInternalId().slice(-12)}`, + spanId: `s_${generateInternalId().slice(-12)}`, + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], +}); + +describe("snapshot store crash boundaries", () => { + containerTest( + "afterPgBeforeRedis: the run converges and burns at most one attempt", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + // Crash between the Postgres commit and the Redis append of ONE transition. + faults.arm("afterPgBeforeRedis", { times: 1, runId: run.id }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + faults.disarm("afterPgBeforeRedis"); + + // The boundary was actually reached. Without this the test could pass having proved nothing. + expect(faults.fired("afterPgBeforeRedis")).toBe(1); + + // Postgres committed the attempt bump; the run is not stuck and not lost. + expect(attempt.run.attemptNumber).toBe(1); + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.attemptNumber).toBe(1); + + // The gap handed the run to the repair job rather than failing the caller. + expect(harness.repairs).toHaveLength(1); + expect(harness.repairs[0]!.runId).toBe(run.id); + + // Reads still resolve: the run's state machine is readable, so nothing hangs. + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + + // The bound: one crash costs at most one attempt number. + expect(pgRun.attemptNumber! - 1).toBeLessThanOrEqual(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "afterRedisBirthBeforePg: no run is created, and the next trigger succeeds", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + + faults.arm("afterRedisBirthBeforePg", { times: 1 }); + await expect( + engine.trigger(triggerArgs("chaos-task", environment), prisma) + ).rejects.toBeInstanceOf(InjectedSnapshotFault); + expect(faults.fired("afterRedisBirthBeforePg")).toBe(1); + + // The harmless state: no run row, so nothing can ever read a run that has no snapshot. + const runsAfterCrash = await prisma.taskRun.count({ + where: { runtimeEnvironmentId: environment.id }, + }); + expect(runsAfterCrash).toBe(0); + + // A crashed birth must not poison the path: the next trigger runs to completion. + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finished.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(finished.attemptNumber).toBe(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "midFlushRetry: a crash during a retry still converges through the repair job", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + // A dead port makes attempt 0 fail FOR REAL, which is the only way the retry boundary is + // reachable: an injected fault at attempt 0 is treated as a dead process and skips the + // retries entirely. Arming midFlushRetry alone would fire nothing and pass for the wrong + // reason, which is what the fired() assertion below catches. + const harness = buildDecoratedStore({ + prisma, + redisOptions: { ...(redisOptions as object), port: 1, retryStrategy: () => null } as never, + mode: "dual-write", + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + + faults.arm("midFlushRetry", { times: 1 }); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + // The birth append failed for real and, before redis-only, that is survivable: Postgres is + // authoritative and the run exists. + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.id).toBe(run.id); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + + // The retry boundary was genuinely reached, not merely armed. + expect(faults.fired("midFlushRetry")).toBeGreaterThanOrEqual(1); + + // The run converges regardless: Postgres holds every snapshot at this dial position. + expect(attempt.run.attemptNumber).toBe(1); + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("EXECUTING"); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "a crash-stalled run rejects a stale snapshot rather than hanging", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + faults.arm("afterPgBeforeRedis", { times: 1, runId: run.id }); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + faults.disarm("afterPgBeforeRedis"); + expect(faults.fired("afterPgBeforeRedis")).toBe(1); + + // Postgres advanced; the Redis head did not. Reads come from Redis, so the caller now holds + // a snapshot id that no longer matches what the read store reports as latest. + // + // The contract is that this SURFACES rather than corrupts: the next operation to validate + // against latest rejects with a stale-snapshot error, which is the same answer a caller gets + // from an ordinary lost race. It does not hang, and it does not silently execute against the + // wrong state. + await expect( + engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }) + ).rejects.toThrow(/Snapshot changed/); + + // The run is still readable and still has a coherent state machine. + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + + // The gap was handed to the repair job, which is the compensator the protocol names. + expect(harness.repairs.length).toBeGreaterThanOrEqual(1); + expect(harness.repairs.some((r) => r.runId === run.id)).toBe(true); + + // And no attempt was burned by the rejection itself: the bound is per crash, and the + // rejected call never reached the attempt bump. + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.attemptNumber ?? 0).toBeLessThanOrEqual(faults.fired("afterPgBeforeRedis")); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "two crashes cost at most two attempts, and the divergence does not amplify", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + // dual-write, so reads still come from Postgres and the run can be driven forward through the + // normal API. That isolates the property under test — how many attempts two crashes cost — + // from the stale-read rejection the previous test covers. + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "dual-write", + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + faults.arm("afterPgBeforeRedis", { times: 2, runId: run.id }); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const crashes = faults.fired("afterPgBeforeRedis"); + expect(crashes).toBeGreaterThanOrEqual(1); + + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + + // pgAttempt - maxLoggedAttempt <= crashCount. Each crash costs at most one attempt, and the + // divergence does not amplify: two crashes never cost three. A flat bound of one was + // refuted by the model check, so the assertion is against the crash count, not a constant. + expect((pgRun.attemptNumber ?? 0) - 1).toBeLessThanOrEqual(crashes); + + // Postgres holds every snapshot at this dial position, so the run still converges. + expect(pgRun.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(harness.repairs.length).toBe(crashes); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts new file mode 100644 index 00000000000..17c186d074d --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts @@ -0,0 +1,304 @@ +// The read gate: the engine's own snapshot flows, run against the decorator with reads served from +// Redis. Same flows, same expectations, different store underneath — the point is that nothing in +// the engine has to know, so no existing suite is modified to make this pass. +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { setTimeout } from "timers/promises"; +import { generateInternalId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { RunEngine } from "../index.js"; +import { buildDecoratedStore, type DecoratedStoreHarness } from "./helpers/decoratedStore.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +function engineOptions(prisma: any, redisOptions: any, harness: DecoratedStoreHarness) { + return { + prisma, + store: harness.store, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerArgs = (taskIdentifier: string, environment: any, n: number) => ({ + number: n, + // A real minted friendly id: the engine converts it back with RunId.fromFriendlyId, which + // rejects anything that is not the prefix plus a cuid body. + friendlyId: RunId.generate().friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `t_gate_${n}`, + spanId: `s_gate_${n}`, + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], +}); + +describe("snapshot store read gate", () => { + containerTest( + "drives a run to completion with every snapshot read served from Redis", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 1), prisma); + await setTimeout(500); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "gate_consumer", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + expect(attempt.run.status).toBe("EXECUTING"); + + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finished.status).toBe("COMPLETED_SUCCESSFULLY"); + + // The gate: the engine read its snapshots, and Redis is what answered. + const fromRedis = harness.reads.filter((r) => r.source === "redis"); + expect(fromRedis.length).toBeGreaterThan(0); + expect(harness.reads.filter((r) => r.source === "postgres")).toEqual([]); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "serves getRunExecutionData from Redis at every step", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 2), prisma); + await setTimeout(500); + + const queued = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(queued); + expect(queued.snapshot.executionStatus).toBe("QUEUED"); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "gate_consumer", + workerQueue: "main", + }); + const pending = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(pending); + expect(pending.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + + await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + const executing = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executing); + expect(executing.snapshot.executionStatus).toBe("EXECUTING"); + expect(executing.run.attemptNumber).toBe(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "keeps the environment boundary on a snapshot read", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 3), prisma); + await setTimeout(500); + + // Scoped to its own environment the run reads normally. + const own = await engine.getRunExecutionData({ + runId: run.id, + environmentId: environment.id, + }); + assertNonNullable(own); + + // Scoped to any other environment the run must not leak across the tenant boundary. The + // assertion is parity rather than a fixed shape: whatever Postgres answers for this call, + // Redis has to answer the same, or the boundary behaves differently once reads move over. + const foreignEnvironmentId = generateInternalId(); + + const viaRedis = await engine + .getRunExecutionData({ runId: run.id, environmentId: foreignEnvironmentId }) + .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); + + const postgresOnly = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, postgresOnly) as never); + let viaPostgres: unknown; + try { + viaPostgres = await engineOff + .getRunExecutionData({ runId: run.id, environmentId: foreignEnvironmentId }) + .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); + } finally { + await engineOff.quit(); + await postgresOnly.quit(); + } + + expect(viaRedis).toEqual(viaPostgres); + // And whatever that shape is, it must not be the run's data. + expect(viaRedis).not.toMatchObject({ run: { id: run.id } }); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "serves a since-window wider than the cap from Redis", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 4), prisma); + await setTimeout(500); + + const first = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(first); + + // More transitions than the 50-cap, so the window is exercised at its boundary. + for (let i = 0; i < 60; i++) { + await harness.store.createExecutionSnapshot({ + run: { id: run.id, status: "PENDING", attemptNumber: null }, + snapshot: { executionStatus: "QUEUED", description: `filler ${i}` }, + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.project.id, + organizationId: environment.organization.id, + }); + } + + const since = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: first.snapshot.id, + }); + assertNonNullable(since); + + // The newest 50, ascending — the same window Postgres would have produced. + expect(since.length).toBe(50); + expect(since[since.length - 1]!.snapshot.description).toBe("filler 59"); + expect(harness.reads.some((r) => r.source === "redis")).toBe(true); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "falls back to Postgres for a pre-cutover run", + async ({ prisma, redisOptions }) => { + // A run created while the dial was off has no keyspace. Turning reads on must not lose it. + const off = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, off) as never); + + let runId: string; + let environment: any; + try { + environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engineOff, environment, "gate-task"); + const run = await engineOff.trigger(triggerArgs("gate-task", environment, 5), prisma); + runId = run.id; + await setTimeout(500); + } finally { + await engineOff.quit(); + await off.quit(); + } + + const on = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engineOn = new RunEngine(engineOptions(prisma, redisOptions, on) as never); + try { + const data = await engineOn.getRunExecutionData({ runId }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("QUEUED"); + expect(on.reads.some((r) => r.source === "postgres")).toBe(true); + } finally { + await engineOn.quit(); + await on.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts new file mode 100644 index 00000000000..c827e10c753 --- /dev/null +++ b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts @@ -0,0 +1,194 @@ +// One-off generator for the RunStore pass-through base. +// +// The base class is mechanical: 80-odd near-identical forwarders. Generating it removes the chance +// of a hand-typo that no test would catch, and turns "did we miss a method" into a diff rather than +// a review. Re-run after any change to the RunStore interface: +// +// pnpm exec tsx scripts/generateDelegatingRunStore.ts +// +// The interface is scanned directly rather than through the TypeScript compiler API, because +// `require("typescript")` resolves to a stub in this workspace. A parsing miss cannot pass silently: +// delegatingRunStore.test.ts asserts the class and the interface hold exactly the same member set, +// and `implements RunStore` fails typecheck if a method is absent. +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const source = readFileSync(join(root, "src/types.ts"), "utf8"); + +/** Replaces every comment and string body with spaces, so a brace inside one cannot move the depth. */ +function blankCommentsAndStrings(text: string): string { + let out = ""; + let i = 0; + while (i < text.length) { + const two = text.slice(i, i + 2); + if (two === "//") { + const end = text.indexOf("\n", i); + const stop = end === -1 ? text.length : end; + out += " ".repeat(stop - i); + i = stop; + } else if (two === "/*") { + const end = text.indexOf("*/", i + 2); + const stop = end === -1 ? text.length : end + 2; + out += text.slice(i, stop).replace(/[^\n]/g, " "); + i = stop; + } else if (text[i] === '"' || text[i] === "'" || text[i] === "`") { + const quote = text[i]; + let j = i + 1; + while (j < text.length && text[j] !== quote) { + j += text[j] === "\\" ? 2 : 1; + } + out += quote + " ".repeat(Math.max(0, j - i - 1)) + (text[j] ?? ""); + i = j + 1; + } else { + out += text[i]; + i += 1; + } + } + return out; +} + +const blanked = blankCommentsAndStrings(source); + +const declaration = "export interface RunStore {"; +const start = blanked.indexOf(declaration); +if (start === -1) { + throw new Error("export interface RunStore not found in src/types.ts"); +} + +const bodyStart = start + declaration.length; +let depth = 1; +let bodyEnd = bodyStart; +while (bodyEnd < blanked.length && depth > 0) { + const ch = blanked[bodyEnd]; + if (ch === "{") depth += 1; + else if (ch === "}") depth -= 1; + if (depth > 0) bodyEnd += 1; +} +if (depth !== 0) { + throw new Error("unbalanced braces while reading the RunStore interface body"); +} + +const body = blanked.slice(bodyStart, bodyEnd); + +// Members are separated by `;` at nesting depth 0. Only `{}`, `()` and `[]` count towards depth: +// angle brackets cannot, because `=>` in a callback parameter type carries an unmatched `>`. +function splitMembers(text: string): { offset: number; length: number }[] { + const spans: { offset: number; length: number }[] = []; + let level = 0; + let from = 0; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (ch === "{" || ch === "(" || ch === "[") level += 1; + else if (ch === "}" || ch === ")" || ch === "]") level -= 1; + else if (ch === ";" && level === 0) { + spans.push({ offset: from, length: i - from }); + from = i + 1; + } + } + if (text.slice(from).trim().length > 0) { + spans.push({ offset: from, length: text.length - from }); + } + return spans; +} + +const methods: string[] = []; +const readonlyProperties: { name: string; type: string }[] = []; +const mutableProperties: string[] = []; + +for (const span of splitMembers(body)) { + const blankedMember = body.slice(span.offset, span.offset + span.length).trim(); + if (blankedMember.length === 0) continue; + + // A method: an optional name then `(` or `<`. A property: a name then `:`. + const asMethod = /^([A-Za-z_$][\w$]*)\s*\??\s*[(<]/.exec(blankedMember); + if (asMethod) { + methods.push(asMethod[1]); + continue; + } + + const asProperty = /^(readonly\s+)?([A-Za-z_$][\w$]*)\s*(\??)\s*:([\s\S]*)$/.exec(blankedMember); + if (asProperty) { + const [, isReadonly, name, , type] = asProperty; + if (isReadonly) { + readonlyProperties.push({ name, type: type.trim() }); + } else { + mutableProperties.push(name); + } + continue; + } + + throw new Error(`could not classify a RunStore member: ${blankedMember.slice(0, 80)}`); +} + +if (mutableProperties.length > 0) { + // A writable data property cannot be forwarded by a getter alone, so the base would silently hold + // its own copy instead of the delegate's. Handle it by hand before regenerating. + throw new Error( + `RunStore declares writable data properties the generator cannot forward: ${mutableProperties.join(", ")}` + ); +} + +const unique = [...new Set(methods)]; +if (unique.length === 0) { + throw new Error("RunStore declares no methods, which cannot be right"); +} + +const memberNames = [...unique, ...readonlyProperties.map((p) => p.name)]; + +const header = `// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. +// Regenerate after any change to the RunStore interface: +// pnpm exec tsx scripts/generateDelegatingRunStore.ts +`; + +writeFileSync( + join(root, "src/runStoreMethodNames.ts"), + `${header} +// Every method the RunStore interface declares. The decorator suites enumerate this, so a method +// added to the interface and not to the base fails a test instead of becoming a silent hole. +export const RUN_STORE_METHOD_NAMES = [ +${unique.map((n) => ` "${n}",`).join("\n")} +] as const; + +// Data properties the base exposes as getters over the delegate, not as forwarders. +export const RUN_STORE_PROPERTY_NAMES = [ +${readonlyProperties.map((p) => ` "${p.name}",`).join("\n")} +] as const; +` +); + +writeFileSync( + join(root, "src/delegatingRunStore.ts"), + `${header} +// A pass-through over another RunStore. It exists so a decorator can override the handful of methods +// it cares about and inherit the rest, instead of restating 80-odd forwarders alongside real logic. +// +// Arguments and return values are forwarded untouched. The \`any\` signatures carry each method's +// whole overload set through one forwarder, which is the single thing a generated base cannot +// preserve; a subclass that overrides a method restates the real signature there. +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { RunStore } from "./types.js"; + +export class DelegatingRunStore implements RunStore { + constructor(protected readonly delegate: RunStore) {} + +${readonlyProperties + // Indexed access rather than the written type, so the getter needs no import of its own and + // follows the interface if that type is ever changed. + .map( + (p) => ` get ${p.name}(): RunStore["${p.name}"] {\n return this.delegate.${p.name};\n }` + ) + .join("\n\n")}${readonlyProperties.length > 0 ? "\n\n" : ""}${unique + .map( + (n) => ` ${n}(...args: any[]): any {\n return (this.delegate as any).${n}(...args);\n }` + ) + .join("\n\n")} +} +` +); + +console.log( + `generated ${unique.length} forwarders and ${readonlyProperties.length} getters ` + + `(${memberNames.length} members total)` +); diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts new file mode 100644 index 00000000000..07948670b28 --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts @@ -0,0 +1,150 @@ +// A caller-supplied snapshot id must survive into Postgres, so the decorator can own the id and both +// stores hold the same one under dual-write. Absent, Prisma's @default(cuid()) still supplies it. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { setupSnapshotIdFixture } from "./testFixtures/snapshotIdFixture.js"; + +describe("PostgresRunStore caller-supplied snapshot id", () => { + postgresTest("completeAttemptSuccess writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + id, + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const snapshot = await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } }); + expect(snapshot).not.toBeNull(); + expect(snapshot!.runId).toBe(run.id); + }); + + postgresTest("expireRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + id, + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("expireParkedRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + const id = generateInternalId(); + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { + id, + engine: "V2", + executionStatus: "FINISHED", + description: "Parked run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(result.count).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("rescheduleRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot: { + id, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("createExecutionSnapshot writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + const created = await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(created.id).toBe(id); + }); + + postgresTest("an absent id still gets a generated one", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const created = await store.createExecutionSnapshot({ + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(created.id).toMatch(/^c[a-z0-9]{24}$/); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts new file mode 100644 index 00000000000..dda0f483bf5 --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts @@ -0,0 +1,312 @@ +// snapshotWrites: false is the redis-only dial position. Every run mutation still lands; no snapshot +// row is written and no completed-waitpoint join row is inserted. The default stays true, so nothing +// changes for any existing caller. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, +} from "./testFixtures/snapshotIdFixture.js"; + +describe("PostgresRunStore snapshotWrites flag", () => { + postgresTest("defaults to writing snapshots", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + }); + + postgresTest("writes the run mutation but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const updated = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(updated.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("createRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await store.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + }); + + postgresTest("createCancelledRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await store.createCancelledRun({ + data: { + ...buildCreateRunData(runId, env), + status: "CANCELED", + error: { type: "STRING_ERROR", raw: "cancelled" }, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0, + }, + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + }); + + postgresTest("expireRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe( + "EXPIRED" + ); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("expireParkedRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Parked run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(result.count).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("rescheduleRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const delayUntil = new Date(Date.now() + 60_000); + + await store.rescheduleRun(run.id, { + delayUntil, + snapshot: { + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const updated = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(updated.delayUntil?.toISOString()).toBe(delayUntil.toISOString()); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("lockRunToWorker writes the lock but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + + await store.lockRunToWorker(run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: generateInternalId(), + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe( + "DEQUEUED" + ); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("createExecutionSnapshot echoes the input when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + const echoed = await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 2 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.id).toBe(id); + expect(echoed.runId).toBe(run.id); + expect(echoed.executionStatus).toBe("EXECUTING"); + expect(echoed.attemptNumber).toBe(2); + expect(echoed.isValid).toBe(true); + expect(echoed.checkpoint).toBeNull(); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("the echoed row rewrites a DEQUEUED run status", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const echoed = await store.createExecutionSnapshot({ + id: generateInternalId(), + run: { id: run.id, status: "DEQUEUED", attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING", description: "Run was dequeued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.runStatus).toBe("PENDING"); + }); + + postgresTest("the echoed row reports an errored snapshot as invalid", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const echoed = await store.createExecutionSnapshot({ + id: generateInternalId(), + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Stale write" }, + error: "snapshot is not the latest", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.isValid).toBe(false); + expect(echoed.error).toBe("snapshot is not the latest"); + }); + + postgresTest("createExecutionSnapshot needs an id when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await expect( + store.createExecutionSnapshot({ + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }) + ).rejects.toThrow(/snapshotWrites is off/); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index b7d5086431b..0f00f5848fe 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -118,6 +118,13 @@ export type PostgresRunStoreOptions = { maxWait?: number; /** Env-driven P2028-at-acquisition retry config, threaded from the app boundary (IoC). */ transactionStartRetry?: TransactionStartRetryConfig; + /** + * When false the store writes no execution-snapshot rows: every nested `executionSnapshots.create` + * is omitted and `createExecutionSnapshot` echoes its input instead of inserting. Only the + * redis-only dial position sets this, once the Redis store is the sole snapshot writer. + * Defaults to true, so the store behaves exactly as it always has. + */ + snapshotWrites?: boolean; }; // A caller sub-select for a relation: `{ select?, include? }` or `true` for a bare `key: true`. @@ -638,6 +645,7 @@ export class PostgresRunStore implements RunStore { private readonly prisma: RunOpsCapableClient; private readonly readOnlyPrisma: RunOpsCapableClient; private readonly schemaVariant: RunStoreSchemaVariant; + private readonly snapshotWrites: boolean; private readonly maxWait?: number; private readonly transactionStartRetry?: TransactionStartRetryConfig; @@ -650,6 +658,16 @@ export class PostgresRunStore implements RunStore { this.schemaVariant = options.schemaVariant ?? "legacy"; this.maxWait = options.maxWait; this.transactionStartRetry = options.transactionStartRetry; + this.snapshotWrites = options.snapshotWrites ?? true; + } + + /** + * Wraps a nested snapshot create so a single flag removes it everywhere. Prisma treats an absent + * key and `undefined` alike, so spreading an empty object drops the nested write entirely rather + * than sending an empty one. + */ + #nestedSnapshot(create: T): { executionSnapshots: { create: T } } | Record { + return this.snapshotWrites ? { executionSnapshots: { create } } : {}; } // The writer handle in read-client form, so the routing layer can honor a caller-passed client @@ -726,6 +744,8 @@ export class PostgresRunStore implements RunStore { const snapshotCreate = { id: params.snapshot.id, + createdAt: params.snapshot.createdAt, + updatedAt: params.snapshot.createdAt, engine: params.snapshot.engine, executionStatus: params.snapshot.executionStatus, description: params.snapshot.description, @@ -743,7 +763,7 @@ export class PostgresRunStore implements RunStore { const run = (await this.#writeClientWithoutTransaction(tx).taskRun.create({ data: { ...params.data, - executionSnapshots: { create: snapshotCreate }, + ...this.#nestedSnapshot(snapshotCreate), }, })) as TaskRun; return { ...run, associatedWaitpoint: null }; @@ -755,7 +775,7 @@ export class PostgresRunStore implements RunStore { const run = (await c.taskRun.create({ data: { ...params.data, - executionSnapshots: { create: snapshotCreate }, + ...this.#nestedSnapshot(snapshotCreate), }, })) as TaskRun; @@ -772,9 +792,7 @@ export class PostgresRunStore implements RunStore { }, data: { ...params.data, - executionSnapshots: { - create: snapshotCreate, - }, + ...this.#nestedSnapshot(snapshotCreate), associatedWaitpoint: params.associatedWaitpoint ? { create: params.associatedWaitpoint, @@ -813,23 +831,26 @@ export class PostgresRunStore implements RunStore { ): Promise { const client = tx ?? this.prisma; + const snapshotCreate = { + id: params.snapshot.id, + createdAt: params.snapshot.createdAt, + updatedAt: params.snapshot.createdAt, + engine: params.snapshot.engine, + executionStatus: params.snapshot.executionStatus, + description: params.snapshot.description, + runStatus: params.snapshot.runStatus, + environmentId: params.snapshot.environmentId, + environmentType: params.snapshot.environmentType, + projectId: params.snapshot.projectId, + organizationId: params.snapshot.organizationId, + workerId: params.snapshot.workerId, + runnerId: params.snapshot.runnerId, + }; + return client.taskRun.create({ data: { ...params.data, - executionSnapshots: { - create: { - engine: params.snapshot.engine, - executionStatus: params.snapshot.executionStatus, - description: params.snapshot.description, - runStatus: params.snapshot.runStatus, - environmentId: params.snapshot.environmentId, - environmentType: params.snapshot.environmentType, - projectId: params.snapshot.projectId, - organizationId: params.snapshot.organizationId, - workerId: params.snapshot.workerId, - runnerId: params.snapshot.runnerId, - }, - }, + ...this.#nestedSnapshot(snapshotCreate), }, }); } @@ -923,20 +944,21 @@ export class PostgresRunStore implements RunStore { outputType: data.outputType, usageDurationMs: data.usageDurationMs, costInCents: data.costInCents, - executionSnapshots: { - create: { - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - attemptNumber: data.snapshot.attemptNumber, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - workerId: data.snapshot.workerId, - runnerId: data.snapshot.runnerId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + attemptNumber: data.snapshot.attemptNumber, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + workerId: data.snapshot.workerId, + runnerId: data.snapshot.runnerId, + }), }, { select: args.select } ) as Promise>; @@ -1129,18 +1151,19 @@ export class PostgresRunStore implements RunStore { completedAt: data.completedAt, expiredAt: data.expiredAt, error: data.error as Prisma.InputJsonValue, - executionSnapshots: { - create: { - engine: data.snapshot.engine, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: data.snapshot.engine, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + }), }, { select: args.select } ) as Promise>; @@ -1260,42 +1283,46 @@ export class PostgresRunStore implements RunStore { cliVersion: data.cliVersion ?? undefined, maxDurationInSeconds: data.maxDurationInSeconds ?? undefined, maxAttempts: data.maxAttempts ?? undefined, - executionSnapshots: { - create: { - id: data.snapshot.id, - engine: "V2", - executionStatus: "PENDING_EXECUTING", - description: "Run was dequeued for execution", - runStatus: "PENDING", - attemptNumber: data.snapshot.attemptNumber ?? undefined, - previousSnapshotId: data.snapshot.previousSnapshotId, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - checkpointId: data.snapshot.checkpointId ?? undefined, - batchId: data.snapshot.batchId ?? undefined, - // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas. - completedWaitpointOrder: data.snapshot.completedWaitpointOrder, - workerId: data.snapshot.workerId ?? undefined, - runnerId: data.snapshot.runnerId ?? undefined, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: "V2", + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + runStatus: "PENDING", + attemptNumber: data.snapshot.attemptNumber ?? undefined, + previousSnapshotId: data.snapshot.previousSnapshotId, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + checkpointId: data.snapshot.checkpointId ?? undefined, + batchId: data.snapshot.batchId ?? undefined, + // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas. + completedWaitpointOrder: data.snapshot.completedWaitpointOrder, + workerId: data.snapshot.workerId ?? undefined, + runnerId: data.snapshot.runnerId ?? undefined, + }), }, }); - if (dedicated) { - await this.#connectCompletedWaitpoints( - prisma, - data.snapshot.id, - data.snapshot.completedWaitpointIds - ); - } else { - await this.#connectCompletedWaitpointsLegacy( - prisma, - data.snapshot.id, - data.snapshot.completedWaitpointIds - ); + // The join rows link to the snapshot row above. With snapshot writes off there is no such row, + // so inserting them would leave dangling links for a snapshot that only the Redis store holds. + if (this.snapshotWrites) { + if (dedicated) { + await this.#connectCompletedWaitpoints( + prisma, + data.snapshot.id, + data.snapshot.completedWaitpointIds + ); + } else { + await this.#connectCompletedWaitpointsLegacy( + prisma, + data.snapshot.id, + data.snapshot.completedWaitpointIds + ); + } } return result; @@ -1363,18 +1390,19 @@ export class PostgresRunStore implements RunStore { completedAt: data.completedAt, expiredAt: data.expiredAt, error: data.error as Prisma.InputJsonValue, - executionSnapshots: { - create: { - engine: data.snapshot.engine, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: data.snapshot.engine, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + }), }, }); } catch (error) { @@ -1435,21 +1463,21 @@ export class PostgresRunStore implements RunStore { data: { delayUntil: data.delayUntil, ...(data.queueTimestamp !== undefined && { queueTimestamp: data.queueTimestamp }), - ...(data.snapshot && { - executionSnapshots: { - create: { - engine: "V2", - executionStatus: data.snapshot.executionStatus ?? "DELAYED", - description: - data.snapshot.description ?? "Delayed run was rescheduled to a future date", - runStatus: data.snapshot.runStatus ?? "DELAYED", - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, - }), + ...(data.snapshot && + this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: "V2", + executionStatus: data.snapshot.executionStatus ?? "DELAYED", + description: + data.snapshot.description ?? "Delayed run was rescheduled to a future date", + runStatus: data.snapshot.runStatus ?? "DELAYED", + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + })), }, }); } @@ -1969,6 +1997,8 @@ export class PostgresRunStore implements RunStore { prisma: PrismaClientOrTransaction ): Promise> { const { + id, + createdAt, run, snapshot, previousSnapshotId, @@ -1984,10 +2014,58 @@ export class PostgresRunStore implements RunStore { error, } = input; + const completedWaitpointOrder = + completedWaitpoints + ?.filter((c) => c.index !== undefined) + .sort((a, b) => a.index! - b.index!) + .map((w) => w.id) ?? []; + + // Redis-only: no row is written and the decorator owns the document. Echo the input in the shape + // the caller expects, so every caller of this method keeps working while Postgres holds nothing. + if (!this.snapshotWrites) { + if (!id) { + throw new Error( + "PostgresRunStore.createExecutionSnapshot: snapshotWrites is off, so the caller must supply the snapshot id" + ); + } + + const now = createdAt ?? new Date(); + return { + id, + engine: "V2", + executionStatus: snapshot.executionStatus, + description: snapshot.description, + previousSnapshotId: previousSnapshotId ?? null, + runId: run.id, + runStatus: run.status === "DEQUEUED" ? "PENDING" : run.status, + attemptNumber: run.attemptNumber ?? null, + batchId: batchId ?? null, + environmentId, + environmentType, + projectId, + organizationId, + checkpointId: checkpointId ?? null, + workerId: workerId ?? null, + runnerId: runnerId ?? null, + metadata: snapshot.metadata ?? null, + completedWaitpointOrder, + isValid: !error, + error: error ?? null, + createdAt: now, + updatedAt: now, + checkpoint: null, + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { checkpoint: true }; + }>; + } + const dedicated = this.schemaVariant === "dedicated"; const newSnapshot = await prisma.taskRunExecutionSnapshot.create({ data: { + id, + createdAt, + updatedAt: createdAt, engine: "V2", executionStatus: snapshot.executionStatus, description: snapshot.description, @@ -2007,10 +2085,7 @@ export class PostgresRunStore implements RunStore { metadata: snapshot.metadata ?? undefined, // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas, so a // cross-DB (NEW-resident) token can be recorded without a Prisma `connect` existence check. - completedWaitpointOrder: completedWaitpoints - ?.filter((c) => c.index !== undefined) - .sort((a, b) => a.index! - b.index!) - .map((w) => w.id), + completedWaitpointOrder, isValid: !error, error, }, diff --git a/internal-packages/run-store/src/delegatingRunStore.test.ts b/internal-packages/run-store/src/delegatingRunStore.test.ts new file mode 100644 index 00000000000..7065f801d71 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.test.ts @@ -0,0 +1,72 @@ +// The base must forward EVERY RunStore member. A method added to the interface and not to the base +// is a silent hole in the decorator built on top of it, so this suite enumerates the generated name +// lists rather than restating them by hand. Regenerate the base and both lists together: +// pnpm exec tsx scripts/generateDelegatingRunStore.ts +import { describe, expect, it } from "vitest"; +import { DelegatingRunStore } from "./delegatingRunStore.js"; +import { RUN_STORE_METHOD_NAMES, RUN_STORE_PROPERTY_NAMES } from "./runStoreMethodNames.js"; +import type { RunStore } from "./types.js"; + +function recordingDelegate(): { store: RunStore; calls: { name: string; args: unknown[] }[] } { + const calls: { name: string; args: unknown[] }[] = []; + const store: Record = {}; + + for (const name of RUN_STORE_METHOD_NAMES) { + store[name] = (...args: unknown[]) => { + calls.push({ name, args }); + return `result:${name}`; + }; + } + for (const name of RUN_STORE_PROPERTY_NAMES) { + store[name] = `property:${name}`; + } + + return { store: store as unknown as RunStore, calls }; +} + +describe("DelegatingRunStore", () => { + it("forwards every RunStore method to the delegate, arguments untouched", () => { + const { store, calls } = recordingDelegate(); + const base = new DelegatingRunStore(store) as unknown as Record< + string, + (...args: unknown[]) => unknown + >; + + for (const name of RUN_STORE_METHOD_NAMES) { + expect(base[name]("arg-one", "arg-two")).toBe(`result:${name}`); + } + + expect(calls.map((c) => c.name)).toEqual([...RUN_STORE_METHOD_NAMES]); + for (const call of calls) { + expect(call.args).toEqual(["arg-one", "arg-two"]); + } + }); + + it("reads every RunStore data property from the delegate", () => { + const { store } = recordingDelegate(); + const base = new DelegatingRunStore(store) as unknown as Record; + + for (const name of RUN_STORE_PROPERTY_NAMES) { + expect(base[name]).toBe(`property:${name}`); + } + }); + + it("reads a data property live, so a delegate that changes is not cached", () => { + const store = { primaryReadClient: "first" } as unknown as RunStore; + const base = new DelegatingRunStore(store); + + expect(base.primaryReadClient).toBe("first" as unknown); + (store as unknown as Record).primaryReadClient = "second"; + expect(base.primaryReadClient).toBe("second" as unknown); + }); + + it("declares exactly the members the interface declares, and no others", () => { + const own = Object.getOwnPropertyNames(DelegatingRunStore.prototype) + .filter((name) => name !== "constructor") + .sort(); + + const expected = [...RUN_STORE_METHOD_NAMES, ...RUN_STORE_PROPERTY_NAMES].sort(); + + expect(own).toEqual(expected); + }); +}); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts new file mode 100644 index 00000000000..6266ac5abd1 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -0,0 +1,300 @@ +// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. +// Regenerate after any change to the RunStore interface: +// pnpm exec tsx scripts/generateDelegatingRunStore.ts + +// A pass-through over another RunStore. It exists so a decorator can override the handful of methods +// it cares about and inherit the rest, instead of restating 80-odd forwarders alongside real logic. +// +// Arguments and return values are forwarded untouched. The `any` signatures carry each method's +// whole overload set through one forwarder, which is the single thing a generated base cannot +// preserve; a subclass that overrides a method restates the real signature there. +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { RunStore } from "./types.js"; + +export class DelegatingRunStore implements RunStore { + constructor(protected readonly delegate: RunStore) {} + + get primaryReadClient(): RunStore["primaryReadClient"] { + return this.delegate.primaryReadClient; + } + + runInTransaction(...args: any[]): any { + return (this.delegate as any).runInTransaction(...args); + } + + createRun(...args: any[]): any { + return (this.delegate as any).createRun(...args); + } + + createCancelledRun(...args: any[]): any { + return (this.delegate as any).createCancelledRun(...args); + } + + createFailedRun(...args: any[]): any { + return (this.delegate as any).createFailedRun(...args); + } + + startAttempt(...args: any[]): any { + return (this.delegate as any).startAttempt(...args); + } + + completeAttemptSuccess(...args: any[]): any { + return (this.delegate as any).completeAttemptSuccess(...args); + } + + recordRetryOutcome(...args: any[]): any { + return (this.delegate as any).recordRetryOutcome(...args); + } + + requeueRun(...args: any[]): any { + return (this.delegate as any).requeueRun(...args); + } + + recordBulkActionMembership(...args: any[]): any { + return (this.delegate as any).recordBulkActionMembership(...args); + } + + cancelRun(...args: any[]): any { + return (this.delegate as any).cancelRun(...args); + } + + failRunPermanently(...args: any[]): any { + return (this.delegate as any).failRunPermanently(...args); + } + + finalizeRun(...args: any[]): any { + return (this.delegate as any).finalizeRun(...args); + } + + expireRun(...args: any[]): any { + return (this.delegate as any).expireRun(...args); + } + + expireRunsBatch(...args: any[]): any { + return (this.delegate as any).expireRunsBatch(...args); + } + + lockRunToWorker(...args: any[]): any { + return (this.delegate as any).lockRunToWorker(...args); + } + + parkPendingVersion(...args: any[]): any { + return (this.delegate as any).parkPendingVersion(...args); + } + + promotePendingVersionRuns(...args: any[]): any { + return (this.delegate as any).promotePendingVersionRuns(...args); + } + + expireParkedRun(...args: any[]): any { + return (this.delegate as any).expireParkedRun(...args); + } + + suspendForCheckpoint(...args: any[]): any { + return (this.delegate as any).suspendForCheckpoint(...args); + } + + resumeFromCheckpoint(...args: any[]): any { + return (this.delegate as any).resumeFromCheckpoint(...args); + } + + rescheduleRun(...args: any[]): any { + return (this.delegate as any).rescheduleRun(...args); + } + + enqueueDelayedRun(...args: any[]): any { + return (this.delegate as any).enqueueDelayedRun(...args); + } + + rewriteDebouncedRun(...args: any[]): any { + return (this.delegate as any).rewriteDebouncedRun(...args); + } + + updateMetadata(...args: any[]): any { + return (this.delegate as any).updateMetadata(...args); + } + + clearIdempotencyKey(...args: any[]): any { + return (this.delegate as any).clearIdempotencyKey(...args); + } + + pushTags(...args: any[]): any { + return (this.delegate as any).pushTags(...args); + } + + pushRealtimeStream(...args: any[]): any { + return (this.delegate as any).pushRealtimeStream(...args); + } + + findRun(...args: any[]): any { + return (this.delegate as any).findRun(...args); + } + + findRunOrThrow(...args: any[]): any { + return (this.delegate as any).findRunOrThrow(...args); + } + + findRunOnPrimary(...args: any[]): any { + return (this.delegate as any).findRunOnPrimary(...args); + } + + findRunOrThrowOnPrimary(...args: any[]): any { + return (this.delegate as any).findRunOrThrowOnPrimary(...args); + } + + findRuns(...args: any[]): any { + return (this.delegate as any).findRuns(...args); + } + + findRunsByIds(...args: any[]): any { + return (this.delegate as any).findRunsByIds(...args); + } + + findRunsByIdempotencyKeys(...args: any[]): any { + return (this.delegate as any).findRunsByIdempotencyKeys(...args); + } + + createBatchTaskRunItem(...args: any[]): any { + return (this.delegate as any).createBatchTaskRunItem(...args); + } + + findLatestExecutionSnapshot(...args: any[]): any { + return (this.delegate as any).findLatestExecutionSnapshot(...args); + } + + findExecutionSnapshot(...args: any[]): any { + return (this.delegate as any).findExecutionSnapshot(...args); + } + + findManyExecutionSnapshots(...args: any[]): any { + return (this.delegate as any).findManyExecutionSnapshots(...args); + } + + createExecutionSnapshot(...args: any[]): any { + return (this.delegate as any).createExecutionSnapshot(...args); + } + + findSnapshotCompletedWaitpointIds(...args: any[]): any { + return (this.delegate as any).findSnapshotCompletedWaitpointIds(...args); + } + + findSnapshotCompletedWaitpointIdsWithPresence(...args: any[]): any { + return (this.delegate as any).findSnapshotCompletedWaitpointIdsWithPresence(...args); + } + + findWaitpointConnectedRunIds(...args: any[]): any { + return (this.delegate as any).findWaitpointConnectedRunIds(...args); + } + + findWaitpointCompletedSnapshotIds(...args: any[]): any { + return (this.delegate as any).findWaitpointCompletedSnapshotIds(...args); + } + + blockRunWithWaitpointEdges(...args: any[]): any { + return (this.delegate as any).blockRunWithWaitpointEdges(...args); + } + + countPendingWaitpoints(...args: any[]): any { + return (this.delegate as any).countPendingWaitpoints(...args); + } + + countPendingWaitpointsWithPresence(...args: any[]): any { + return (this.delegate as any).countPendingWaitpointsWithPresence(...args); + } + + createWaitpoint(...args: any[]): any { + return (this.delegate as any).createWaitpoint(...args); + } + + upsertWaitpoint(...args: any[]): any { + return (this.delegate as any).upsertWaitpoint(...args); + } + + findWaitpoint(...args: any[]): any { + return (this.delegate as any).findWaitpoint(...args); + } + + findWaitpointOnPrimary(...args: any[]): any { + return (this.delegate as any).findWaitpointOnPrimary(...args); + } + + findManyWaitpoints(...args: any[]): any { + return (this.delegate as any).findManyWaitpoints(...args); + } + + updateWaitpoint(...args: any[]): any { + return (this.delegate as any).updateWaitpoint(...args); + } + + updateManyWaitpoints(...args: any[]): any { + return (this.delegate as any).updateManyWaitpoints(...args); + } + + forWaitpointCompletion(...args: any[]): any { + return (this.delegate as any).forWaitpointCompletion(...args); + } + + findManyTaskRunWaitpoints(...args: any[]): any { + return (this.delegate as any).findManyTaskRunWaitpoints(...args); + } + + deleteManyTaskRunWaitpoints(...args: any[]): any { + return (this.delegate as any).deleteManyTaskRunWaitpoints(...args); + } + + findTaskRunAttempt(...args: any[]): any { + return (this.delegate as any).findTaskRunAttempt(...args); + } + + createTaskRunCheckpoint(...args: any[]): any { + return (this.delegate as any).createTaskRunCheckpoint(...args); + } + + createBatchTaskRun(...args: any[]): any { + return (this.delegate as any).createBatchTaskRun(...args); + } + + updateBatchTaskRun(...args: any[]): any { + return (this.delegate as any).updateBatchTaskRun(...args); + } + + findBatchTaskRunById(...args: any[]): any { + return (this.delegate as any).findBatchTaskRunById(...args); + } + + findBatchTaskRunByFriendlyId(...args: any[]): any { + return (this.delegate as any).findBatchTaskRunByFriendlyId(...args); + } + + findBatchTaskRunByIdempotencyKey(...args: any[]): any { + return (this.delegate as any).findBatchTaskRunByIdempotencyKey(...args); + } + + updateManyBatchTaskRun(...args: any[]): any { + return (this.delegate as any).updateManyBatchTaskRun(...args); + } + + countBatchTaskRunItems(...args: any[]): any { + return (this.delegate as any).countBatchTaskRunItems(...args); + } + + updateManyBatchTaskRunItems(...args: any[]): any { + return (this.delegate as any).updateManyBatchTaskRunItems(...args); + } + + findManyBatchTaskRunItems(...args: any[]): any { + return (this.delegate as any).findManyBatchTaskRunItems(...args); + } + + findBatchTaskRunItem(...args: any[]): any { + return (this.delegate as any).findBatchTaskRunItem(...args); + } + + upsertWaitpointTag(...args: any[]): any { + return (this.delegate as any).upsertWaitpointTag(...args); + } + + findManyWaitpointTags(...args: any[]): any { + return (this.delegate as any).findManyWaitpointTags(...args); + } +} diff --git a/internal-packages/run-store/src/index.ts b/internal-packages/run-store/src/index.ts index 3717dc01527..16fa696ee43 100644 --- a/internal-packages/run-store/src/index.ts +++ b/internal-packages/run-store/src/index.ts @@ -3,3 +3,8 @@ export * from "./PostgresRunStore.js"; export * from "./runOpsStore.js"; export * from "./readReplicaClient.js"; export * from "./redisSnapshotStore.js"; +export * from "./delegatingRunStore.js"; +export * from "./taskRunExecutionSnapshotStore.js"; +export * from "./snapshotEntry.js"; +export * from "./snapshotFaultInjection.js"; +export * from "./snapshotOrphanSweeper.js"; diff --git a/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts new file mode 100644 index 00000000000..f1394be597e --- /dev/null +++ b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts @@ -0,0 +1,193 @@ +// getExecutionSnapshotsSince resolves its cursor to a createdAt before it asks for the window, so +// the snapshot id is gone by then and getSince cannot serve it. This read takes the cursor instead, +// and has to agree with the Postgres read it stands in for — same-millisecond blind spot included. +import { describe, expect } from "vitest"; +import { redisTest } from "@internal/testcontainers"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function entry(runId: string, id: string, createdAt: string): SnapshotEntryInput { + return { + id, + engine: "V2", + executionStatus: "EXECUTING", + description: "d", + runId, + runStatus: "EXECUTING", + createdAt, + environmentId: "env_1", + environmentType: "DEVELOPMENT", + projectId: "proj_1", + organizationId: "org_1", + }; +} + +const at = (seconds: number) => new Date(Date.UTC(2026, 0, 1, 0, 0, seconds)).toISOString(); + +async function seed( + store: RedisSnapshotStore, + runId: string, + stamps: { id: string; createdAt: string }[] +): Promise { + for (const [index, stamp] of stamps.entries()) { + await store.append({ + entry: entry(runId, stamp.id, stamp.createdAt), + kind: index === 0 ? "birth" : "transition", + isTerminal: false, + }); + } +} + +describe("getSinceCreatedAt", () => { + redisTest( + "returns only entries newer than the cursor, oldest first", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_window"; + await seed( + store, + runId, + [0, 1, 2, 3, 4].map((n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(1)); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + // Ascending, matching what the engine hands its caller after its own reverse(). + expect(result.entries.map((e) => e.id)).toEqual(["snap_2", "snap_3", "snap_4"]); + } finally { + await store.quit(); + } + } + ); + + redisTest("misses when the run has no keyspace", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + // A miss is the coexistence path: the caller falls back to Postgres for a pre-cutover run. + expect((await store.getSinceCreatedAt("run_absent", at(0))).kind).toBe("miss"); + } finally { + await store.quit(); + } + }); + + redisTest("returns an empty hit when nothing is newer", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_nothing_newer"; + await seed(store, runId, [{ id: "snap_0", createdAt: at(0) }]); + + const result = await store.getSinceCreatedAt(runId, at(5)); + + // A hit, not a miss: Redis owns this run, so the caller must not fall back and re-read + // Postgres for a window it already answered. + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries).toEqual([]); + } finally { + await store.quit(); + } + }); + + redisTest("drops a same-millisecond neighbour, as Postgres does", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_same_ms"; + const shared = at(1); + await seed(store, runId, [ + { id: "snap_0", createdAt: at(0) }, + { id: "snap_1a", createdAt: shared }, + { id: "snap_1b", createdAt: shared }, + { id: "snap_2", createdAt: at(2) }, + ]); + + const result = await store.getSinceCreatedAt(runId, shared); + + // Postgres serves this window with `createdAt: { gt: cursor }`, which drops both same-ms + // entries. Returning snap_1b here would be more correct than Postgres and would therefore + // read as divergence in compare mode. + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries.map((e) => e.id)).toEqual(["snap_2"]); + } finally { + await store.quit(); + } + }); + + redisTest("caps the window at the limit, keeping the newest", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_capped"; + await seed( + store, + runId, + Array.from({ length: 60 }, (_, n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(0), { limit: 50 }); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries).toHaveLength(50); + // The engine takes the NEWEST 50 and reverses, so the window ends at the newest entry. + expect(result.entries[result.entries.length - 1]!.id).toBe("snap_59"); + expect(result.entries[0]!.id).toBe("snap_10"); + } finally { + await store.quit(); + } + }); + + redisTest("scans no further than the answer", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_deep_history"; + await seed( + store, + runId, + Array.from({ length: 400 }, (_, n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const started = Date.now(); + const result = await store.getSinceCreatedAt(runId, at(394), { limit: 50 }); + const elapsed = Date.now() - started; + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries.map((e) => e.id)).toEqual([ + "snap_395", + "snap_396", + "snap_397", + "snap_398", + "snap_399", + ]); + // The walk stops at the cursor rather than reading the run's history. The bound is generous + // on purpose: it fails on a full scan of 400 entries, not on ordinary timing noise. + expect(elapsed).toBeLessThan(1_000); + } finally { + await store.quit(); + } + }); + + redisTest("scopes the window to an environment", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_env_scoped"; + await seed(store, runId, [ + { id: "snap_0", createdAt: at(0) }, + { id: "snap_1", createdAt: at(1) }, + ]); + + const foreign = await store.getSinceCreatedAt(runId, at(0), { environmentId: "env_other" }); + + expect(foreign.kind).toBe("hit"); + if (foreign.kind !== "hit") return; + expect(foreign.entries).toEqual([]); + } finally { + await store.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 2964959c4fe..e36179c6041 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -472,6 +472,64 @@ export class RedisSnapshotStore { }); } + /** + * The same window as {@link getSince}, addressed by a createdAt cursor instead of a snapshot id. + * + * `getExecutionSnapshotsSince` resolves its cursor to a createdAt before it asks for the window, + * so the snapshot id is gone by the time this call is made and `getSince` cannot serve it. The + * cursor is exclusive and keeps Postgres's same-millisecond blind spot, so the two reads agree. + */ + async getSinceCreatedAt( + runId: string, + createdAt: Date | string, + opts?: { environmentId?: string; limit?: number } + ): Promise { + return this.#timed("getSinceCreatedAt", async () => { + const k = snapshotKeys(runId); + const limit = opts?.limit ?? this.sinceLimit; + const cursor = typeof createdAt === "string" ? createdAt : createdAt.toISOString(); + + const reply = await this.redis.readSnapshotsSinceCreatedAt( + k.e, + k.idx, + k.cur, + k.seq, + cursor, + String(limit) + ); + if (reply === null) return { kind: "miss" }; + + const headOrder = reply[1] ?? ""; + const rows: SnapshotRead[] = []; + // Tracks whether the Lua-chosen head row (always the first, i === 2) survives the env filter, + // so headOrder is never attributed to a different, surviving row. + let headSurvived = false; + for (let i = 2; i + 3 < reply.length; i += 4) { + const decoded = this.#decode( + [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], + opts?.environmentId, + runId, + false + ); + if (decoded) { + rows.push(decoded); + if (i === 2) headSurvived = true; + } + } + + rows.reverse(); + const head = headSurvived ? rows[rows.length - 1] : undefined; + const headWaitpointIds = decodeWaitpointIds(head !== undefined, head ? headOrder : ""); + if (head) { + head.completedWaitpointIds = headWaitpointIds; + if (head.cycle) { + this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); + } + } + return { kind: "hit", entries: rows, headWaitpointIds }; + }); + } + #checkCycleMismatch(runId: string, count: number, orderLength: number): void { if (orderLength === count) return; this.metrics?.recordCycleMismatch(); @@ -682,6 +740,66 @@ export class RedisSnapshotStore { `, }); + this.redis.defineCommand("readSnapshotsSinceCreatedAt", { + numberOfKeys: 4, + lua: ` + ${PRELUDE} + local cursor = ARGV[1] + local limit = tonumber(ARGV[2]) + + -- A run with no keyspace is a MISS, so the caller falls back to Postgres. A run that has one + -- and nothing newer is an empty HIT, so it does not fall back for a window it owns. + if redis.call('EXISTS', eKey) == 0 then return nil end + + -- STRICTLY greater than the cursor, and same-millisecond entries are dropped. Postgres + -- serves this window with createdAt > cursor and drops them too; a Redis read that is more + -- correct than the Postgres read shows up as divergence in compare mode. + -- + -- createdAt is always toISOString() output, one fixed-width UTC format, so a lexicographic + -- compare is a chronological compare. Walking newest-first lets the scan stop at the first + -- entry at or before the cursor, which makes its length the length of the ANSWER rather + -- than the length of the run's history. + local out = { '', '' } + local headId = nil + local offset = 0 + local page = limit + local done = false + + while not done do + local ids = redis.call('ZREVRANGE', idxKey, offset, offset + page - 1) + if #ids == 0 then break end + + for i = 1, #ids do + local id = ids[i] + local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') + if vals[1] then + local createdAt = cjson.decode(vals[1])['createdAt'] + if not createdAt or createdAt <= cursor then + done = true + break + end + if not headId then headId = id end + out[#out + 1] = id + out[#out + 1] = vals[1] + out[#out + 1] = vals[2] or '' + out[#out + 1] = vals[3] or '' + if (#out - 2) / 4 >= limit then + done = true + break + end + end + end + + offset = offset + page + end + + if headId then + out[2] = orderFor(redis.call('HGET', eKey, headId .. '#c')) + end + return out + `, + }); + this.redis.defineCommand("readSnapshotsSince", { numberOfKeys: 4, lua: ` @@ -780,6 +898,15 @@ declare module "@internal/redis" { id: string, callback?: Callback ): Result; + readSnapshotsSinceCreatedAt( + eKey: string, + idxKey: string, + curKey: string, + seqKey: string, + createdAtCursor: string, + limit: string, + callback?: Callback + ): Result; readSnapshotsSince( eKey: string, idxKey: string, diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts new file mode 100644 index 00000000000..d3dd0868676 --- /dev/null +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -0,0 +1,81 @@ +// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. +// Regenerate after any change to the RunStore interface: +// pnpm exec tsx scripts/generateDelegatingRunStore.ts + +// Every method the RunStore interface declares. The decorator suites enumerate this, so a method +// added to the interface and not to the base fails a test instead of becoming a silent hole. +export const RUN_STORE_METHOD_NAMES = [ + "runInTransaction", + "createRun", + "createCancelledRun", + "createFailedRun", + "startAttempt", + "completeAttemptSuccess", + "recordRetryOutcome", + "requeueRun", + "recordBulkActionMembership", + "cancelRun", + "failRunPermanently", + "finalizeRun", + "expireRun", + "expireRunsBatch", + "lockRunToWorker", + "parkPendingVersion", + "promotePendingVersionRuns", + "expireParkedRun", + "suspendForCheckpoint", + "resumeFromCheckpoint", + "rescheduleRun", + "enqueueDelayedRun", + "rewriteDebouncedRun", + "updateMetadata", + "clearIdempotencyKey", + "pushTags", + "pushRealtimeStream", + "findRun", + "findRunOrThrow", + "findRunOnPrimary", + "findRunOrThrowOnPrimary", + "findRuns", + "findRunsByIds", + "findRunsByIdempotencyKeys", + "createBatchTaskRunItem", + "findLatestExecutionSnapshot", + "findExecutionSnapshot", + "findManyExecutionSnapshots", + "createExecutionSnapshot", + "findSnapshotCompletedWaitpointIds", + "findSnapshotCompletedWaitpointIdsWithPresence", + "findWaitpointConnectedRunIds", + "findWaitpointCompletedSnapshotIds", + "blockRunWithWaitpointEdges", + "countPendingWaitpoints", + "countPendingWaitpointsWithPresence", + "createWaitpoint", + "upsertWaitpoint", + "findWaitpoint", + "findWaitpointOnPrimary", + "findManyWaitpoints", + "updateWaitpoint", + "updateManyWaitpoints", + "forWaitpointCompletion", + "findManyTaskRunWaitpoints", + "deleteManyTaskRunWaitpoints", + "findTaskRunAttempt", + "createTaskRunCheckpoint", + "createBatchTaskRun", + "updateBatchTaskRun", + "findBatchTaskRunById", + "findBatchTaskRunByFriendlyId", + "findBatchTaskRunByIdempotencyKey", + "updateManyBatchTaskRun", + "countBatchTaskRunItems", + "updateManyBatchTaskRunItems", + "findManyBatchTaskRunItems", + "findBatchTaskRunItem", + "upsertWaitpointTag", + "findManyWaitpointTags", +] as const; + +// Data properties the base exposes as getters over the delegate, not as forwarders. +export const RUN_STORE_PROPERTY_NAMES = ["primaryReadClient"] as const; diff --git a/internal-packages/run-store/src/snapshotEntry.parity.test.ts b/internal-packages/run-store/src/snapshotEntry.parity.test.ts new file mode 100644 index 00000000000..635160fd831 --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.parity.test.ts @@ -0,0 +1,522 @@ +// The entry is built from a write site's input while Postgres builds the row from the same input by +// a different code path. This suite is the only thing that keeps those two paths equal, so it covers +// every one of the ten physical snapshot-create sites in PostgresRunStore. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, +} from "./snapshotEntry.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +/** + * NOTE ON createdAt. An earlier version of this suite built the expected entry with + * `createdAt: row.createdAt`, reading the value off the row it was checking and then asserting the + * two matched. That can never fail, and it hid a real divergence: seven of the eight write sites + * stamped the entry from the app clock while Postgres stamped its own column default, so the two + * stores held different instants for one snapshot. + * + * Every case now mints ONE instant, passes it to the store call, and gives the builder the same + * value. The row must carry it because the write site forwards it. A write site that stops + * forwarding the caller's instant fails here. + * + * Compares only what the entry claims. The Redis model carries no `updatedAt` and no join rows, and + * it holds `createdAt` as an ISO string, so those are checked separately or not at all. + */ +function assertParity(entry: SnapshotEntryInput, row: Record) { + expect(row.id).toBe(entry.id); + expect(row.runId).toBe(entry.runId); + expect(row.engine).toBe(entry.engine); + expect(row.executionStatus).toBe(entry.executionStatus); + expect(row.description).toBe(entry.description); + expect(row.runStatus).toBe(entry.runStatus); + expect(row.environmentId).toBe(entry.environmentId); + expect(row.environmentType).toBe(entry.environmentType); + expect(row.projectId).toBe(entry.projectId); + expect(row.organizationId).toBe(entry.organizationId); + expect(row.attemptNumber ?? undefined).toBe(entry.attemptNumber ?? undefined); + expect(row.previousSnapshotId ?? undefined).toBe(entry.previousSnapshotId ?? undefined); + expect(row.batchId ?? undefined).toBe(entry.batchId ?? undefined); + expect(row.checkpointId ?? undefined).toBe(entry.checkpointId ?? undefined); + expect(row.workerId ?? undefined).toBe(entry.workerId ?? undefined); + expect(row.runnerId ?? undefined).toBe(entry.runnerId ?? undefined); + expect(row.isValid).toBe(entry.error === undefined); + expect((row.createdAt as Date).toISOString()).toBe(entry.createdAt); + // Write-once rows: both columns hold the one instant, so a Prisma-stamped updatedAt would drift. + expect((row.updatedAt as Date).toISOString()).toBe(entry.createdAt); +} + +/** Five minutes in the past, so a database default could never coincide with it. */ +const independentStamp = new Date(Date.now() - 5 * 60 * 1000); + +function birthSnapshot(id: string, env: SnapshotFixtureEnv) { + return { + id, + createdAt: independentStamp, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("entry to Postgres row parity", () => { + postgresTest("createRun, legacy schema", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = birthSnapshot(id, env); + + await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createRun with an associated waitpoint, legacy schema", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = birthSnapshot(id, env); + + await store.createRun({ + data: buildCreateRunData(runId, env), + snapshot, + associatedWaitpoint: { + id: generateInternalId(), + friendlyId: `waitpoint_${runId.slice(-12)}`, + type: "RUN", + status: "PENDING", + idempotencyKey: generateInternalId(), + userProvidedIdempotencyKey: false, + projectId: env.projectId, + environmentId: env.id, + }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createRun carries the worker and runner ids", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const { workerId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = { ...birthSnapshot(id, env), workerId, runnerId: "runner_1" }; + + await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createCancelledRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = { + ...birthSnapshot(id, env), + executionStatus: "FINISHED" as const, + description: "Run was cancelled", + runStatus: "CANCELED" as const, + }; + + await store.createCancelledRun({ + data: { + ...buildCreateRunData(runId, env), + status: "CANCELED", + error: { type: "STRING_ERROR", raw: "cancelled" }, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0, + }, + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("completeAttemptSuccess", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromCompletion({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("expireRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromExpire({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("expireParkedRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Parked run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot, + }); + + expect(result.count).toBe(1); + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromExpire({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("rescheduleRun with every default applied", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromReschedule({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("rescheduleRun with every value supplied", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + executionStatus: "QUEUED" as const, + runStatus: "PENDING" as const, + description: "custom reschedule", + }; + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromReschedule({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("lockRunToWorker", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const previous = await store.createExecutionSnapshot({ + run: { id: run.id, status: "PENDING", attemptNumber: null }, + snapshot: { executionStatus: "QUEUED", description: "Run was queued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + previousSnapshotId: previous.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }; + + await store.lockRunToWorker(run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromLock({ id, runId: run.id, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createExecutionSnapshot, the standalone site", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + createdAt: independentStamp, + run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 2 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + const created = await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(created.id).toBe(id); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), + row + ); + }); + + postgresTest("createExecutionSnapshot rewrites a DEQUEUED run status", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + createdAt: independentStamp, + run: { id: run.id, status: "DEQUEUED" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING" as const, description: "Run was dequeued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.runStatus).toBe("PENDING"); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), + row + ); + }); + + postgresTest("createExecutionSnapshot with an error is invalid in both", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + createdAt: independentStamp, + run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Stale write" }, + error: "snapshot is not the latest", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.isValid).toBe(false); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), + row + ); + }); +}); + +// The clock-provenance guard. Independent of the builders above: it asserts that what Postgres +// stores is the instant the CALLER supplied, not one the database chose. Without this, a snapshot +// has two different creation times depending on which store answers, the compared field can never +// reach zero divergence, and the since-window cursor resolved from one store misfilters the window +// walked in the other. +describe("createdAt provenance", () => { + postgresTest("Postgres stores the caller's instant, not its own", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + // Far enough from now that a database default could never coincide with it. + const stamp = new Date(Date.now() - 5 * 60 * 1000); + + await store.createExecutionSnapshot({ + id, + createdAt: stamp, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.toISOString()).toBe(stamp.toISOString()); + expect(row.updatedAt.toISOString()).toBe(stamp.toISOString()); + }); + + postgresTest("an absent instant still takes the database default", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const before = new Date(Date.now() - 1000); + + // Mode off supplies nothing, so Postgres must behave exactly as it always has. This is what + // keeps the merge test true. + await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.getTime()).toBeGreaterThan(before.getTime()); + }); + + postgresTest("a nested write site stores the caller's instant too", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const stamp = new Date(Date.now() - 5 * 60 * 1000); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + id, + createdAt: stamp, + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.toISOString()).toBe(stamp.toISOString()); + expect(row.updatedAt.toISOString()).toBe(stamp.toISOString()); + }); +}); diff --git a/internal-packages/run-store/src/snapshotEntry.test.ts b/internal-packages/run-store/src/snapshotEntry.test.ts new file mode 100644 index 00000000000..cc624d690a9 --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.test.ts @@ -0,0 +1,185 @@ +// These mappings are values Postgres derives rather than receives. If either side changes and the +// other does not, dual-write silently stores two different documents for one snapshot. The parity +// suite next to this file checks the same thing against a real Postgres row; this one pins the +// rules on their own, so a failure says which rule broke. +import { describe, expect, it } from "vitest"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, + isTerminalEntry, +} from "./snapshotEntry.js"; + +const ctx = { id: "snap_1", runId: "run_1", createdAt: new Date("2026-08-24T00:00:00.000Z") }; +const scope = { + environmentId: "env_1", + environmentType: "DEVELOPMENT" as const, + projectId: "proj_1", + organizationId: "org_1", +}; + +describe("snapshotEntry derived values", () => { + it("rewrites a DEQUEUED run status to PENDING", () => { + const entry = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "DEQUEUED", attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING", description: "d" }, + ...scope, + }); + + expect(entry.runStatus).toBe("PENDING"); + }); + + it("keeps every other run status unchanged", () => { + const entry = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + ...scope, + }); + + expect(entry.runStatus).toBe("EXECUTING"); + }); + + it("applies the lock site's hard-coded values", () => { + const entry = entryFromLock(ctx, { + id: "snap_1", + previousSnapshotId: "snap_0", + attemptNumber: 2, + completedWaitpointIds: [], + completedWaitpointOrder: [], + ...scope, + }); + + expect(entry.executionStatus).toBe("PENDING_EXECUTING"); + expect(entry.description).toBe("Run was dequeued for execution"); + expect(entry.runStatus).toBe("PENDING"); + expect(entry.engine).toBe("V2"); + expect(entry.previousSnapshotId).toBe("snap_0"); + expect(entry.attemptNumber).toBe(2); + }); + + it("applies the reschedule defaults", () => { + const entry = entryFromReschedule(ctx, { ...scope }); + + expect(entry.executionStatus).toBe("DELAYED"); + expect(entry.runStatus).toBe("DELAYED"); + expect(entry.description).toBe("Delayed run was rescheduled to a future date"); + }); + + it("prefers a supplied reschedule value over the default", () => { + const entry = entryFromReschedule(ctx, { + ...scope, + executionStatus: "QUEUED", + runStatus: "PENDING", + description: "custom", + }); + + expect(entry.executionStatus).toBe("QUEUED"); + expect(entry.runStatus).toBe("PENDING"); + expect(entry.description).toBe("custom"); + }); + + it("sets engine V2 on a completion, which Postgres leaves to the column default", () => { + const entry = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...scope, + }); + + expect(entry.engine).toBe("V2"); + }); + + it("carries a null completion attemptNumber through as null", () => { + const entry = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: null, + ...scope, + }); + + expect(entry.attemptNumber).toBeNull(); + }); + + it("omits an absent optional rather than writing undefined into the document", () => { + const entry = entryFromExpire(ctx, { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + ...scope, + }); + + expect(Object.keys(entry)).not.toContain("workerId"); + expect(Object.keys(entry)).not.toContain("attemptNumber"); + expect(JSON.parse(JSON.stringify(entry))).toEqual(entry); + }); + + it("reports a FINISHED entry as terminal and any other as not", () => { + const finished = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...scope, + }); + const running = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + ...scope, + }); + + expect(isTerminalEntry(finished)).toBe(true); + expect(isTerminalEntry(running)).toBe(false); + }); + + it("serialises createdAt as an ISO string", () => { + const entry = entryFromReschedule(ctx, { ...scope }); + + expect(entry.createdAt).toBe("2026-08-24T00:00:00.000Z"); + }); + + it("carries the birth site's worker and runner ids", () => { + const entry = entryFromCreateRun(ctx, { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + workerId: "worker_1", + runnerId: "runner_1", + ...scope, + }); + + expect(entry.workerId).toBe("worker_1"); + expect(entry.runnerId).toBe("runner_1"); + expect(entry.executionStatus).toBe("RUN_CREATED"); + }); + + it("never sets the reserved completedWaitpoints field", () => { + const built = [ + entryFromReschedule(ctx, { ...scope }), + entryFromLock(ctx, { + id: "snap_1", + previousSnapshotId: "snap_0", + completedWaitpointIds: ["w_1"], + completedWaitpointOrder: ["w_1"], + ...scope, + }), + entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + completedWaitpoints: [{ id: "w_1", index: 0 }], + ...scope, + }), + ]; + + // The append script mints the pointer as a sidecar field, and rejects an entry that carries one. + for (const entry of built) { + expect(entry.completedWaitpoints).toBeUndefined(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotEntry.ts b/internal-packages/run-store/src/snapshotEntry.ts new file mode 100644 index 00000000000..748a95f6d1a --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.ts @@ -0,0 +1,168 @@ +// Builds the Redis entry for each execution-snapshot write site, from that site's own INPUT. +// +// Not from the delegate's return value: no nested write site includes the snapshot in what it +// returns. `createRun` returns the run, `expireParkedRun` returns a count, and the rest return a +// selected `TaskRun`. That means every value Postgres derives rather than receives has to be +// reproduced here, and snapshotEntry.parity.test.ts is what keeps the two sides from drifting. +import type { TaskRunStatus } from "@trigger.dev/database"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import type { + CompletionSnapshotInput, + CreateExecutionSnapshotInput, + CreateRunSnapshotInput, + ExpireSnapshotInput, + LockSnapshotInput, + RescheduleSnapshotInput, +} from "./types.js"; + +export type EntryBuildContext = { id: string; runId: string; createdAt: Date }; + +/** + * PostgresRunStore.#createExecutionSnapshot rewrites DEQUEUED to PENDING, because older runners + * reject DEQUEUED on a snapshot. Every site that can carry that status must rewrite it identically. + */ +function snapshotRunStatus(status: TaskRunStatus): string { + return status === "DEQUEUED" ? "PENDING" : status; +} + +function base(ctx: EntryBuildContext) { + return { + id: ctx.id, + runId: ctx.runId, + createdAt: ctx.createdAt.toISOString(), + engine: "V2" as const, + }; +} + +export function entryFromCreateRun( + ctx: EntryBuildContext, + snapshot: CreateRunSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +/** + * `completeAttemptSuccess` writes no `engine` column, so Postgres applies the schema default of + * `V2`. The entry states it, because SnapshotEntryInput requires the field. + */ +export function entryFromCompletion( + ctx: EntryBuildContext, + snapshot: CompletionSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + attemptNumber: snapshot.attemptNumber, + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +/** Serves both `expireRun` and `expireParkedRun`; the two write identical snapshot columns. */ +export function entryFromExpire( + ctx: EntryBuildContext, + snapshot: ExpireSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + }; +} + +/** PostgresRunStore.rescheduleRun supplies these three defaults inline, so the entry repeats them. */ +export function entryFromReschedule( + ctx: EntryBuildContext, + snapshot: RescheduleSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus ?? "DELAYED", + description: snapshot.description ?? "Delayed run was rescheduled to a future date", + runStatus: snapshotRunStatus(snapshot.runStatus ?? "DELAYED"), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + }; +} + +/** PostgresRunStore.#lockRunToWorker hard-codes the status, description and run status. */ +export function entryFromLock( + ctx: EntryBuildContext, + snapshot: LockSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + runStatus: "PENDING", + previousSnapshotId: snapshot.previousSnapshotId, + ...(snapshot.attemptNumber !== undefined && { attemptNumber: snapshot.attemptNumber }), + ...(snapshot.batchId !== undefined && { batchId: snapshot.batchId }), + ...(snapshot.checkpointId !== undefined && { checkpointId: snapshot.checkpointId }), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +export function entryFromCreateExecutionSnapshot( + ctx: EntryBuildContext, + input: CreateExecutionSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: input.snapshot.executionStatus, + description: input.snapshot.description, + runStatus: snapshotRunStatus(input.run.status), + ...(input.run.attemptNumber !== undefined && + input.run.attemptNumber !== null && { attemptNumber: input.run.attemptNumber }), + ...(input.previousSnapshotId !== undefined && { previousSnapshotId: input.previousSnapshotId }), + ...(input.batchId !== undefined && { batchId: input.batchId }), + environmentId: input.environmentId, + environmentType: input.environmentType, + projectId: input.projectId, + organizationId: input.organizationId, + ...(input.checkpointId !== undefined && { checkpointId: input.checkpointId }), + ...(input.workerId !== undefined && { workerId: input.workerId }), + ...(input.runnerId !== undefined && { runnerId: input.runnerId }), + ...(input.snapshot.metadata !== undefined && + input.snapshot.metadata !== null && { metadata: input.snapshot.metadata }), + ...(input.error !== undefined && { error: input.error }), + }; +} + +/** + * A terminal entry is what makes the append script apply the completion TTL. FINISHED is the only + * terminal execution status; the run-level status is not consulted, because a run reaches its + * terminal state through a FINISHED snapshot in every path. + */ +export function isTerminalEntry(entry: SnapshotEntryInput): boolean { + return entry.executionStatus === "FINISHED"; +} diff --git a/internal-packages/run-store/src/snapshotFaultInjection.ts b/internal-packages/run-store/src/snapshotFaultInjection.ts new file mode 100644 index 00000000000..83abf43b179 --- /dev/null +++ b/internal-packages/run-store/src/snapshotFaultInjection.ts @@ -0,0 +1,45 @@ +// Test-only seam for the execution-snapshot write protocol. +// +// The protocol's correctness claim is about crashes: whatever the write order leaves behind at each +// boundary must be a state the existing stall-and-repair machinery heals. Proving that needs a crash +// at an exact point, which is what an injector gives. Production never sets one, so each boundary +// costs one optional call. + +/** The three points a crash can land between the two stores' writes. */ +export type SnapshotFaultBoundary = + /** A transition: Postgres has committed and the Redis append has not started. */ + | "afterPgBeforeRedis" + /** A birth: the Redis append has landed and the Postgres insert has not started. */ + | "afterRedisBirthBeforePg" + /** Inside the append retry loop, after at least one attempt has failed. */ + | "midFlushRetry"; + +export type SnapshotFaultInjector = ( + boundary: SnapshotFaultBoundary, + context: { runId: string; snapshotId: string } +) => void; + +/** + * Thrown by a test injector. The write path tells this apart from a real append failure, because an + * injected fault models a process that died rather than a call that failed. The two write paths then + * do different things with it, and both differ from a real failure: + * + * - A transition skips its remaining retries, hands the run to the repair job, and does NOT rethrow. + * Postgres has already committed, so the caller must not see an error. + * - A birth rethrows, so the Postgres insert never runs and the crash leaves an orphaned keyspace + * with no run row, which is the harmless state that ordering exists to produce. + * - A real append failure is retried, and only then handed to the repair job. + */ +export class InjectedSnapshotFault extends Error { + readonly boundary: SnapshotFaultBoundary; + + constructor(boundary: SnapshotFaultBoundary) { + super(`injected snapshot fault at ${boundary}`); + this.name = "InjectedSnapshotFault"; + this.boundary = boundary; + } +} + +export function isInjectedFault(error: unknown): error is InjectedSnapshotFault { + return error instanceof InjectedSnapshotFault; +} diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts new file mode 100644 index 00000000000..fc3bdcca39f --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -0,0 +1,386 @@ +// The sweep deletes whole keyspaces, so most of these tests are about what it must NOT touch: a live +// run, a young orphan, and any batch whose Postgres lookup did not come back. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore, snapshotKeys } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; +const ORPHAN_AGE_MS = 60 * 60 * 1000; + +function birthEntry(runId: string, env: SnapshotFixtureEnv, createdAt: Date, terminal = false) { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: terminal ? ("FINISHED" as const) : ("RUN_CREATED" as const), + description: "Run was created", + runStatus: terminal ? ("CANCELED" as const) : ("PENDING" as const), + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + return entryFromCreateRun({ id: snapshot.id, runId, createdAt }, snapshot); +} + +describe("SnapshotOrphanSweeper", () => { + containerTest( + "rule 1 expires a terminal run whose keyspace never got one", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // Non-terminal append, so no expiry is ever set — the lost-TTL-set case. + await store.append({ + entry: birthEntry(runId, env, new Date()), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const keys = snapshotKeys(runId); + expect(await probe.pttl(keys.e)).toBe(-1); + + const result = await sweeper.sweep(); + + expect(result.expired).toBe(1); + expect(result.deleted).toBe(0); + for (const key of [keys.e, keys.idx, keys.cur, keys.seq]) { + const ttl = await probe.pttl(key); + expect(ttl).toBeGreaterThan(0); + expect(ttl).toBeLessThanOrEqual(COMPLETED_TTL_MS); + } + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "rule 1 leaves a keyspace that already has an expiry alone", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A healthy terminal append sets the completion TTL itself. + await store.append({ + entry: birthEntry(runId, env, new Date(), true), + kind: "birth", + isTerminal: true, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "CANCELED" }, + }); + + const before = await probe.pttl(snapshotKeys(runId).e); + const result = await sweeper.sweep(); + + expect(result.expired).toBe(0); + expect(result.skipped).toBe(1); + const after = await probe.pttl(snapshotKeys(runId).e); + // Not extended: the sweep must not keep resetting a countdown that is already running. + expect(after).toBeLessThanOrEqual(before); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "rule 2 deletes a keyspace with no run row, cycle keys included", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + // The crashed birth: an entry, no Postgres run, non-terminal so no expiry. + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "transition", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, + }); + + const cyclesBefore = await probe.keys(`snap:{${runId}}:wp:*`); + expect(cyclesBefore.length).toBeGreaterThan(0); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(1); + const keys = snapshotKeys(runId); + for (const key of [keys.e, keys.idx, keys.cur, keys.seq]) { + expect(await probe.exists(key)).toBe(0); + } + expect(await probe.keys(`snap:{${runId}}:wp:*`)).toEqual([]); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("rule 2 spares a young orphan", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // Written just now: the Postgres insert of a healthy birth may still be in flight. + await store.append({ + entry: birthEntry(runId, env, new Date()), + kind: "birth", + isTerminal: false, + }); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.skipped).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "never touches a live run, however old its keyspace", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const ancient = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000); + + // A run waiting on an untimed token can sit non-terminal for weeks. Reaping it would drop + // live state, which is the failure this rule exists to avoid. + await store.append({ + entry: birthEntry(runId, env, ancient), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "WAITING_TO_RESUME" }, + }); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.expired).toBe(0); + expect(result.skipped).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + expect(await probe.pttl(snapshotKeys(runId).e)).toBe(-1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("a dry run reports but changes nothing", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const orphan = generateInternalId(); + const terminal = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + await store.append({ entry: birthEntry(orphan, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(terminal, env, new Date()), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(terminal, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const result = await sweeper.sweep({ dryRun: true }); + + expect(result.deleted).toBe(1); + expect(result.expired).toBe(1); + expect(await probe.exists(snapshotKeys(orphan).e)).toBe(1); + expect(await probe.pttl(snapshotKeys(terminal).e)).toBe(-1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "skips a batch whose run lookup failed, and deletes nothing", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const failing = { + findRunsByIds: async () => { + throw new Error("run lookup unavailable"); + }, + } as unknown as RunStore; + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: failing, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); + + // A failed lookup says nothing about whether the run exists, and rule 2 deletes a whole + // keyspace. The sweep must resolve rather than throw, and must reap nothing. + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.skipped).toBeGreaterThan(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "discovers and reaps a keyspace whose entries are all invalid", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + // The append script writes `cur` and indexes the entry only when it is valid, so a keyspace + // whose entries all carry an error has neither. A sweep that discovers keyspaces by their + // `cur` key would never see this one, and neither rule would ever apply to it. + await store.append({ + entry: { ...birthEntry(runId, env, old), error: "stale write" }, + kind: "birth", + isTerminal: false, + }); + + const keys = snapshotKeys(runId); + expect(await probe.exists(keys.e)).toBe(1); + expect(await probe.exists(keys.cur)).toBe(0); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(1); + expect(await probe.exists(keys.e)).toBe(0); + expect(await probe.exists(keys.seq)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("processes every keyspace across batches", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + const orphans = Array.from({ length: 5 }, () => generateInternalId()); + for (const runId of orphans) { + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); + } + + const result = await sweeper.sweep({ batchSize: 2 }); + + expect(result.deleted).toBe(5); + for (const runId of orphans) { + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + } + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts new file mode 100644 index 00000000000..6ce5c574c2f --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -0,0 +1,322 @@ +// Reaps snapshot keyspaces that no healthy path will ever clean up. +// +// Two rules, because neither can see what the other leaves behind: +// +// 1. The run is terminal in Postgres but its keyspace never got the completion expiry — a +// terminal append whose TTL-set was lost. Applying the expiry now reaps it on the same +// schedule a healthy terminal append would have. +// 2. The keyspace has no Postgres run row at all, and is older than a threshold — a crashed +// birth. It is non-terminal so it carries no expiry, and it has no run row, so rule 1 can +// never match it. Without this rule that leak has no bound. +// +// Nothing schedules this. The engine's worker is what has to run it, and run-store cannot reach the +// engine, so the wiring belongs to the ticket that owns production construction. +import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { Logger } from "@trigger.dev/core/logger"; +import type { TaskRunStatus } from "@trigger.dev/database"; +import { snapshotKeys } from "./redisSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +/** + * Mirrors the engine's `finalStatuses`. run-store cannot import from run-engine — the dependency + * runs the other way — so the list is duplicated and a parity test in run-engine asserts the copy + * stays equal to the original. + */ +export const FINAL_RUN_STATUSES: readonly TaskRunStatus[] = [ + "CANCELED", + "INTERRUPTED", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", +]; + +const FINAL = new Set(FINAL_RUN_STATUSES); + +/** Comfortably above run-creation latency, so a birth in flight is never mistaken for an orphan. */ +const DEFAULT_ORPHAN_AGE_MS = 24 * 60 * 60 * 1000; +const DEFAULT_BATCH_SIZE = 1000; + +export type SweepResult = { + /** Keyspaces examined. */ + scanned: number; + /** Rule 1: terminal runs whose keyspace was given the completion expiry. */ + expired: number; + /** Rule 2: keyspaces with no run row, deleted. */ + deleted: number; + /** Left alone: a live run, a young orphan, or a batch whose Postgres lookup failed. */ + skipped: number; +}; + +export type SnapshotOrphanSweeperOptions = { + /** + * The sweep opens its own connection rather than borrowing the store's, so a long scan can never + * stall a hot-path client. + */ + redisOptions: RedisOptions; + /** + * Resolved through the run store, not a raw client. Under the run-ops split a run row can live on + * either database, and only the store knows which — a raw lookup would report a live run as an + * orphan and delete its keyspace. + */ + runStore: RunStore; + completedTtlMs: number; + orphanAgeMs?: number; + keyPrefix?: string; + logger?: Logger; +}; + +export class SnapshotOrphanSweeper { + readonly #redis: Redis; + readonly #runStore: RunStore; + readonly #completedTtlMs: number; + readonly #orphanAgeMs: number; + readonly #prefix: string; + readonly #logger: Logger; + #quit?: Promise; + + constructor(options: SnapshotOrphanSweeperOptions) { + this.#logger = options.logger ?? new Logger("SnapshotOrphanSweeper", "debug"); + this.#runStore = options.runStore; + this.#completedTtlMs = options.completedTtlMs; + this.#orphanAgeMs = options.orphanAgeMs ?? DEFAULT_ORPHAN_AGE_MS; + this.#prefix = options.keyPrefix ?? "snap:"; + this.#redis = createRedisClient(options.redisOptions, { + onError: (error) => this.#logger.error("SnapshotOrphanSweeper redis client error", { error }), + }); + } + + async quit(): Promise { + if (!this.#quit) { + this.#quit = this.#redis.quit().then( + () => undefined, + () => undefined + ); + } + await this.#quit; + } + + /** + * One full pass over the keyspace. `dryRun` reports what it would do and changes nothing. + */ + async sweep(opts?: { batchSize?: number; dryRun?: boolean }): Promise { + const batchSize = opts?.batchSize ?? DEFAULT_BATCH_SIZE; + const dryRun = opts?.dryRun ?? false; + const result: SweepResult = { scanned: 0, expired: 0, deleted: 0, skipped: 0 }; + + let cursor = "0"; + do { + // Match on the entry hash, not on `cur`. The append script writes `cur` only when the entry + // is valid, so a keyspace whose entries are all invalid would never be discovered and would + // leak with no expiry, which is the same unbounded leak rule 2 exists to close. `e` is + // written by every append. + const [next, keys] = await this.#redis.scan( + cursor, + "MATCH", + `${this.#prefix}{*}:e`, + "COUNT", + batchSize + ); + cursor = next; + + const runIds = [...new Set(keys.map((key) => this.#runIdFrom(key)).filter(isString))]; + if (runIds.length === 0) continue; + + await this.#sweepBatch(runIds, dryRun, result); + } while (cursor !== "0"); + + this.#logger.log("SnapshotOrphanSweeper pass complete", { ...result, dryRun }); + return result; + } + + async #sweepBatch(runIds: string[], dryRun: boolean, result: SweepResult): Promise { + result.scanned += runIds.length; + + let rows: Map; + try { + rows = (await this.#runStore.findRunsByIds(runIds, { + select: { id: true, status: true }, + })) as unknown as Map; + } catch (error) { + // Never reap on an unknown answer. A lookup that failed says nothing about whether the run + // exists, and rule 2 deletes a whole keyspace. + this.#logger.error("SnapshotOrphanSweeper skipped a batch after a failed run lookup", { + count: runIds.length, + error, + }); + result.skipped += runIds.length; + return; + } + + for (const runId of runIds) { + const run = rows.get(runId); + + if (!run) { + await this.#applyRuleTwo(runId, dryRun, result); + continue; + } + + if (!FINAL.has(run.status)) { + // A live run. A SUSPENDED run can legitimately wait for weeks, so this is never touched. + result.skipped += 1; + continue; + } + + await this.#applyRuleOne(runId, dryRun, result); + } + } + + /** Rule 1: a terminal run whose keyspace never received the completion expiry. */ + async #applyRuleOne(runId: string, dryRun: boolean, result: SweepResult): Promise { + const keys = await this.#allKeys(runId); + if (keys.length === 0) { + result.skipped += 1; + return; + } + + const ttls = await Promise.all(keys.map((key) => this.#redis.pttl(key))); + // -1 is "exists, no expiry". Anything already counting down was set by a healthy append. + if (!ttls.some((ttl) => ttl === -1)) { + result.skipped += 1; + return; + } + + if (!dryRun) { + const pipeline = this.#redis.pipeline(); + for (const key of keys) { + pipeline.pexpire(key, this.#completedTtlMs); + } + await pipeline.exec(); + } + + result.expired += 1; + } + + /** Rule 2: a keyspace with no run row at all, past the age threshold. */ + async #applyRuleTwo(runId: string, dryRun: boolean, result: SweepResult): Promise { + const keys = await this.#allKeys(runId); + if (keys.length === 0) { + result.skipped += 1; + return; + } + + const age = await this.#newestEntryAgeMs(runId); + if (age === undefined || age < this.#orphanAgeMs) { + // Either the keyspace carries no readable timestamp, or a birth may still be in flight. + result.skipped += 1; + return; + } + + if (!dryRun) { + await this.#redis.del(...keys); + } + + result.deleted += 1; + } + + /** + * Every key for one run: the four core keys plus each wait-cycle key. + * + * The cycle keys are enumerated from the `c` high-water field on the seq hash, which the append + * script mints densely with HINCRBY, so 1..high covers every wp key that was ever written. This + * is the same source the store's own terminal-expiry loop uses. + * + * It deliberately does NOT use `KEYS`. That command iterates the whole database and blocks while + * it does, and a hash tag routes a key without scoping the scan, so one sweep pass over a batch + * would issue a full keyspace scan per run. + * + * The trade-off: if the seq hash is evicted while a wp key survives, `high` reads 0 and that + * orphaned cycle key is left behind. That is the right way to be wrong here. Leaving one small + * key costs bytes, where scanning the keyspace to find it costs every hot-path client latency on + * every pass. + */ + async #allKeys(runId: string): Promise { + const core = snapshotKeys(runId); + + const high = Number((await this.#redis.hget(core.seq, "c")) ?? "0"); + const cycles: string[] = []; + for (let n = 1; n <= high; n++) { + cycles.push(`${this.#prefix}{${runId}}:wp:${n}`); + } + + const candidates = [core.e, core.idx, core.cur, core.seq, ...cycles]; + + // One round trip for the whole set, rather than one per candidate. + const pipeline = this.#redis.pipeline(); + for (const key of candidates) { + pipeline.exists(key); + } + const replies = await pipeline.exec(); + + return candidates.filter((_key, index) => replies?.[index]?.[1] === 1); + } + + /** + * Age of the newest entry, so a keyspace still being written to is never treated as an orphan. + * The newest is the right end: an old first entry says nothing about whether the run is dead. + */ + async #newestEntryAgeMs(runId: string): Promise { + const core = snapshotKeys(runId); + + const newest = await this.#redis.zrevrange(core.idx, 0, 0); + const id = newest[0]; + + const raw = id + ? await this.#redis.hget(core.e, id) + : // The index holds valid entries only, so an all-invalid keyspace has an empty index. Fall + // back to the newest instant in the entry hash, or that keyspace is never old enough to + // reap and the leak survives the scan fix above. + await this.#newestRawFromEntries(core.e); + + if (!raw) return undefined; + + try { + const createdAt = (JSON.parse(raw) as { createdAt?: string }).createdAt; + if (!createdAt) return undefined; + const parsed = Date.parse(createdAt); + return Number.isNaN(parsed) ? undefined : Date.now() - parsed; + } catch { + return undefined; + } + } + + /** + * The newest entry document in the hash, by its own createdAt. Only reached for a keyspace with + * no index, which is rare, so the whole-hash read is acceptable where it would not be on the + * indexed path. + */ + async #newestRawFromEntries(eKey: string): Promise { + const all = await this.#redis.hgetall(eKey); + let newestRaw: string | undefined; + let newestAt = -Infinity; + + for (const [field, raw] of Object.entries(all)) { + // Sidecar fields hang off the entry ids as `#s` and `#c`; skip them. + if (field.includes("#")) continue; + try { + const at = Date.parse((JSON.parse(raw) as { createdAt?: string }).createdAt ?? ""); + if (!Number.isNaN(at) && at > newestAt) { + newestAt = at; + newestRaw = raw; + } + } catch { + continue; + } + } + + return newestRaw; + } + + #runIdFrom(key: string): string | undefined { + const open = key.indexOf("{"); + const close = key.indexOf("}", open + 1); + if (open === -1 || close === -1 || close === open + 1) return undefined; + return key.slice(open + 1, close); + } +} + +function isString(value: string | undefined): value is string { + return typeof value === "string"; +} diff --git a/internal-packages/run-store/src/snapshotReadShapes.test.ts b/internal-packages/run-store/src/snapshotReadShapes.test.ts new file mode 100644 index 00000000000..258731a60b5 --- /dev/null +++ b/internal-packages/run-store/src/snapshotReadShapes.test.ts @@ -0,0 +1,151 @@ +// A matcher that is too loose is the dangerous failure: it answers a query Redis cannot actually +// serve, and the caller gets a wrong answer rather than a slow one. So most of these tests are +// about what must NOT match. +import { describe, expect, it } from "vitest"; +import { matchSinceCursorLookup, matchSinceWindow } from "./snapshotReadShapes.js"; + +const cursorArgs = { + where: { id: "snap_1", runId: "run_1" }, + select: { createdAt: true }, +}; + +const windowArgs = { + where: { runId: "run_1", isValid: true, createdAt: { gt: new Date("2026-08-24T00:00:00Z") } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, +}; + +describe("matchSinceCursorLookup", () => { + it("matches the engine's since-cursor lookup", () => { + expect(matchSinceCursorLookup(cursorArgs)).toEqual({ id: "snap_1", runId: "run_1" }); + }); + + it("carries an environment scope when present", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, environmentId: "env_1" }, + select: { createdAt: true }, + }) + ).toEqual({ id: "snap_1", runId: "run_1", environmentId: "env_1" }); + }); + + it("ignores keys explicitly set to undefined", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, environmentId: undefined }, + select: { createdAt: true }, + }) + ).toEqual({ id: "snap_1", runId: "run_1" }); + }); + + it("refuses a selection of anything but createdAt", () => { + expect( + matchSinceCursorLookup({ where: cursorArgs.where, select: { description: true } }) + ).toBeUndefined(); + expect( + matchSinceCursorLookup({ + where: cursorArgs.where, + select: { createdAt: true, description: true }, + }) + ).toBeUndefined(); + }); + + it("refuses a where with no run id, because there is no keyspace to look in", () => { + expect( + matchSinceCursorLookup({ where: { id: "snap_1" }, select: { createdAt: true } }) + ).toBeUndefined(); + }); + + it("refuses an unknown where key", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, isValid: true }, + select: { createdAt: true }, + }) + ).toBeUndefined(); + }); + + it("refuses an unknown top-level key", () => { + expect( + matchSinceCursorLookup({ ...cursorArgs, orderBy: { createdAt: "desc" } }) + ).toBeUndefined(); + }); + + it("refuses anything that is not an argument object", () => { + expect(matchSinceCursorLookup(undefined)).toBeUndefined(); + expect(matchSinceCursorLookup(null)).toBeUndefined(); + expect(matchSinceCursorLookup("where")).toBeUndefined(); + expect(matchSinceCursorLookup([cursorArgs])).toBeUndefined(); + }); +}); + +describe("matchSinceWindow", () => { + it("matches the engine's window query", () => { + expect(matchSinceWindow(windowArgs)).toEqual({ + runId: "run_1", + createdAt: new Date("2026-08-24T00:00:00Z"), + take: 50, + }); + }); + + it("carries an environment scope when present", () => { + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, environmentId: "env_1" }, + }) + ).toMatchObject({ environmentId: "env_1" }); + }); + + it("refuses a query that also wants the completed waitpoints", () => { + // The engine omits them on purpose to avoid an N x M read. An include that asks for them is a + // different query, and answering it from this path would return them empty. + expect( + matchSinceWindow({ + ...windowArgs, + include: { checkpoint: true, completedWaitpoints: true }, + }) + ).toBeUndefined(); + }); + + it("refuses ascending order", () => { + expect(matchSinceWindow({ ...windowArgs, orderBy: { createdAt: "asc" } })).toBeUndefined(); + }); + + it("refuses a window that does not filter to valid entries", () => { + expect( + matchSinceWindow({ ...windowArgs, where: { ...windowArgs.where, isValid: false } }) + ).toBeUndefined(); + }); + + it("refuses a cursor that is not a strict greater-than on a Date", () => { + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, createdAt: { gte: new Date() } }, + }) + ).toBeUndefined(); + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, createdAt: { gt: "2026-08-24T00:00:00Z" } }, + }) + ).toBeUndefined(); + }); + + it("refuses a missing take", () => { + const { take: _dropped, ...withoutTake } = windowArgs; + expect(matchSinceWindow(withoutTake)).toBeUndefined(); + }); + + it("refuses an unknown where key", () => { + expect( + matchSinceWindow({ ...windowArgs, where: { ...windowArgs.where, batchId: "batch_1" } }) + ).toBeUndefined(); + }); + + it("refuses an unknown top-level key", () => { + expect(matchSinceWindow({ ...windowArgs, skip: 10 })).toBeUndefined(); + }); +}); diff --git a/internal-packages/run-store/src/snapshotReadShapes.ts b/internal-packages/run-store/src/snapshotReadShapes.ts new file mode 100644 index 00000000000..31dda17f7f0 --- /dev/null +++ b/internal-packages/run-store/src/snapshotReadShapes.ts @@ -0,0 +1,95 @@ +// Shape matchers for the two generic Prisma-args snapshot reads. +// +// `findExecutionSnapshot` and `findManyExecutionSnapshots` take arbitrary Prisma arguments, and a +// key-value store cannot answer an arbitrary query. Only three production call sites exist, all in +// the engine's executionSnapshotSystem, and both generic ones send a single fixed shape. So these +// matchers recognise exactly those shapes and return undefined for anything else, which sends the +// call to Postgres. +// +// Each matcher rejects an argument object carrying any key it does not know about. A query that has +// drifted must fall through and be answered correctly by Postgres, never answered approximately +// from Redis. + +type Unknown = Record; + +function isPlainObject(value: unknown): value is Unknown { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** True when `value` has exactly `allowed` keys, ignoring keys explicitly set to undefined. */ +function hasOnlyKeys(value: Unknown, allowed: string[]): boolean { + const present = Object.keys(value).filter((k) => value[k] !== undefined); + return present.every((k) => allowed.includes(k)); +} + +function isString(value: unknown): value is string { + return typeof value === "string"; +} + +export type SinceCursorLookup = { id: string; runId: string; environmentId?: string }; + +/** + * Step 1 of `getExecutionSnapshotsSince`: resolve a known snapshot id to its createdAt. + * + * { where: { id, runId, environmentId? }, select: { createdAt: true } } + */ +export function matchSinceCursorLookup(args: unknown): SinceCursorLookup | undefined { + if (!isPlainObject(args) || !hasOnlyKeys(args, ["where", "select"])) return undefined; + + const { where, select } = args; + if (!isPlainObject(where) || !isPlainObject(select)) return undefined; + if (!hasOnlyKeys(where, ["id", "runId", "environmentId"])) return undefined; + if (!hasOnlyKeys(select, ["createdAt"]) || select.createdAt !== true) return undefined; + if (!isString(where.id) || !isString(where.runId)) return undefined; + if (where.environmentId !== undefined && !isString(where.environmentId)) return undefined; + + return { + id: where.id, + runId: where.runId, + ...(isString(where.environmentId) && { environmentId: where.environmentId }), + }; +} + +export type SinceWindow = { + runId: string; + createdAt: Date; + take: number; + environmentId?: string; +}; + +/** + * Step 2 of `getExecutionSnapshotsSince`: the capped window after a createdAt cursor. + * + * { where: { runId, isValid: true, createdAt: { gt }, environmentId? }, + * include: { checkpoint: true }, orderBy: { createdAt: "desc" }, take: N } + * + * The engine deliberately omits completedWaitpoints from the include to avoid an N x M read, so an + * include asking for them is a different query and is not matched. + */ +export function matchSinceWindow(args: unknown): SinceWindow | undefined { + if (!isPlainObject(args) || !hasOnlyKeys(args, ["where", "include", "orderBy", "take"])) { + return undefined; + } + + const { where, include, orderBy, take } = args; + if (!isPlainObject(where) || !isPlainObject(include) || !isPlainObject(orderBy)) return undefined; + if (typeof take !== "number") return undefined; + + if (!hasOnlyKeys(where, ["runId", "isValid", "createdAt", "environmentId"])) return undefined; + if (!isString(where.runId) || where.isValid !== true) return undefined; + if (where.environmentId !== undefined && !isString(where.environmentId)) return undefined; + + if (!hasOnlyKeys(include, ["checkpoint"]) || include.checkpoint !== true) return undefined; + if (!hasOnlyKeys(orderBy, ["createdAt"]) || orderBy.createdAt !== "desc") return undefined; + + const cursor = where.createdAt; + if (!isPlainObject(cursor) || !hasOnlyKeys(cursor, ["gt"])) return undefined; + if (!(cursor.gt instanceof Date)) return undefined; + + return { + runId: where.runId, + createdAt: cursor.gt, + take, + ...(isString(where.environmentId) && { environmentId: where.environmentId }), + }; +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts new file mode 100644 index 00000000000..12ab76a0f26 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts @@ -0,0 +1,285 @@ +// A birth writes Redis FIRST. The order is proved by crashing between the two writes and observing +// which side survived: an orphaned key with no run row is the harmless state, and a run with no +// snapshot at all is the one the order exists to prevent. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + opts?: { + mode?: SnapshotStoreMode; + faults?: ConstructorParameters[1]["faults"]; + unreachableRedis?: boolean; + } +) { + // An unreachable port makes every append throw for real, which is the failure the retry loop and + // the mode-dependent refusal are about. A fault injector cannot stand in: an injected fault means + // "the process died", and the two are handled differently on purpose. + const redis = new RedisSnapshotStore({ + redisOptions: opts?.unreachableRedis + ? ({ ...(redisOptions as object), port: 1, retryStrategy: () => null } as never) + : redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + }); + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "dual-write", + ...(opts?.faults && { faults: opts.faults }), + } + ); + + return { decorated, redis }; +} + +function birthSnapshot(id: string, env: SnapshotFixtureEnv) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +function cancelledData(runId: string, env: SnapshotFixtureEnv) { + return { + ...buildCreateRunData(runId, env), + status: "CANCELED" as const, + error: { type: "STRING_ERROR", raw: "cancelled" } as never, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0 as const, + }; +} + +describe("birth write ordering", () => { + containerTest("writes Redis then Postgres", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(snapshotId, env), + }); + + const read = await redis.getLatest(runId); + expect(read).not.toBeNull(); + expect(read!.entry.id).toBe(snapshotId); + expect(read!.entry.executionStatus).toBe("RUN_CREATED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + containerTest("mints an id when the caller supplies none", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const { id: _omitted, ...withoutId } = birthSnapshot(generateInternalId(), env); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot: withoutId }); + + const read = await redis.getLatest(runId); + expect(read).not.toBeNull(); + // The same minted id must reach both stores, or the comparator chases a difference that is + // not real. + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { runId } }); + expect(read!.entry.id).toBe(row.id); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a crash after the Redis append leaves an orphan key and no run", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterRedisBirthBeforePg") throw new InjectedSnapshotFault(boundary); + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await expect( + decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }) + ).rejects.toBeInstanceOf(InjectedSnapshotFault); + + // The harmless state: a keyspace nothing can reach, and no run that lacks a snapshot. + expect(await redis.getLatest(runId)).not.toBeNull(); + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "creates the run anyway when the birth append fails before redis-only", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + mode: "dual-write", + unreachableRedis: true, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + // Postgres is authoritative in every position before redis-only, so a Redis outage must not + // stop runs being created. + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(snapshotId, env), + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "refuses to create the run when the birth append fails at redis-only", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + mode: "redis-only", + unreachableRedis: true, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + // At redis-only Postgres writes no snapshot, so a run created without its Redis birth would + // have no snapshot anywhere. Failing before the run row exists lets the caller retry clean. + await expect( + decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }) + ).rejects.toThrow(); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest("createCancelledRun writes Redis first", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + await decorated.createCancelledRun({ + data: cancelledData(runId, env), + snapshot: { + ...birthSnapshot(snapshotId, env), + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + }, + }); + + const read = await redis.getLatest(runId); + expect(read!.entry.id).toBe(snapshotId); + expect(read!.entry.executionStatus).toBe("FINISHED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a born-terminal run gets the completion expiry immediately", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createCancelledRun({ + data: cancelledData(runId, env), + snapshot: { + ...birthSnapshot(generateInternalId(), env), + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + }, + }); + + // A born-terminal run never transitions again, so the completion TTL has to be applied by + // the birth itself or the keyspace never expires. + const nonTerminal = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(nonTerminal, env), + snapshot: birthSnapshot(generateInternalId(), env), + }); + + const terminal = await redis.getLatest(runId); + const alive = await redis.getLatest(nonTerminal); + expect(terminal).not.toBeNull(); + expect(alive).not.toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { mode: "off" }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts new file mode 100644 index 00000000000..975c68c371a --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts @@ -0,0 +1,94 @@ +// Mode off is the merge-test position: the decorator must be indistinguishable from its delegate and +// must not touch Redis at all. A Redis store whose every member throws proves the second half, and +// enumerating the generated name list proves the first for every method rather than a chosen few. +import { describe, expect, it } from "vitest"; +import { RUN_STORE_METHOD_NAMES } from "./runStoreMethodNames.js"; +import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +function explodingRedisStore(): RedisSnapshotStore { + return new Proxy({} as RedisSnapshotStore, { + get(_target, prop) { + return () => { + throw new Error(`the Redis store must not be called at mode off, but ${String(prop)} was`); + }; + }, + }); +} + +function recordingDelegate(): { store: RunStore; calls: string[] } { + const calls: string[] = []; + const store: Record = {}; + + for (const name of RUN_STORE_METHOD_NAMES) { + store[name] = (...args: unknown[]) => { + calls.push(name); + return `result:${name}`; + }; + } + + return { store: store as unknown as RunStore, calls }; +} + +describe("TaskRunExecutionSnapshotStore at mode off", () => { + it("defaults to mode off", () => { + const { store } = recordingDelegate(); + + const decorated = new TaskRunExecutionSnapshotStore(store, { store: explodingRedisStore() }); + + expect(decorated.mode).toBe("off"); + }); + + it("forwards every method to the delegate and never calls Redis", async () => { + const { store, calls } = recordingDelegate(); + const decorated = new TaskRunExecutionSnapshotStore(store, { + store: explodingRedisStore(), + mode: "off", + }) as unknown as Record unknown>; + + for (const name of RUN_STORE_METHOD_NAMES) { + if (name === "runInTransaction") continue; + expect(await decorated[name]("arg-one", "arg-two")).toBe(`result:${name}`); + } + + expect(calls).toEqual(RUN_STORE_METHOD_NAMES.filter((n) => n !== "runInTransaction")); + }); + + it("hands the delegate's own store to a transaction callback", async () => { + const inner = recordingDelegate().store; + let seen: unknown; + const delegate = { + runInTransaction: async ( + _runId: string | undefined, + fn: (store: RunStore, tx: unknown) => Promise + ) => { + await fn(inner, "tx"); + }, + } as unknown as RunStore; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: explodingRedisStore(), + mode: "off", + }); + + await decorated.runInTransaction("run_1", async (store) => { + seen = store; + }); + + expect(seen).toBe(inner); + }); + + it("reports every other dial position as one that writes Redis", () => { + const { store } = recordingDelegate(); + const modes = ["dual-write", "compare", "redis-read", "redis-only"] as const; + + for (const mode of modes) { + const decorated = new TaskRunExecutionSnapshotStore(store, { + store: explodingRedisStore(), + mode, + }); + expect(decorated.mode).toBe(mode); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts new file mode 100644 index 00000000000..12ad2e5fc54 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts @@ -0,0 +1,95 @@ +// The read cohort is pure arithmetic on the run id, so it needs no containers. Keeping it out of the +// container-backed suite also keeps that suite small enough to run reliably. +import { describe, expect, it } from "vitest"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +type CohortProbe = { readsFromRedis(runId: string): boolean }; + +function probe(mode: SnapshotStoreMode, readPercent: number): CohortProbe { + // lazyConnect keeps the client from dialling anything: no read in this suite reaches the store. + const redis = new RedisSnapshotStore({ + redisOptions: { host: "127.0.0.1", port: 1, lazyConnect: true, retryStrategy: () => null }, + completedTtlMs: 1, + }); + + return new TaskRunExecutionSnapshotStore({} as RunStore, { + store: redis, + mode, + readPercent, + }) as unknown as CohortProbe; +} + +const ids = Array.from({ length: 500 }, (_, n) => `run_cohort_${n}_${n * 7919}`); + +describe("the read cohort", () => { + it("reads nothing from Redis before the read positions", () => { + for (const mode of ["off", "dual-write", "compare"] as const) { + const store = probe(mode, 100); + expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); + } + }); + + it("reads everything from Redis at 100 percent", () => { + for (const mode of ["redis-read", "redis-only"] as const) { + const store = probe(mode, 100); + expect(ids.every((id) => store.readsFromRedis(id))).toBe(true); + } + }); + + it("reads nothing from Redis at 0 percent", () => { + const store = probe("redis-read", 0); + expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); + }); + + it("gives one run the same answer every time", () => { + // A run that changed store between two reads of one poll could show the log going backwards. + const store = probe("redis-read", 50); + + for (const id of ids.slice(0, 50)) { + const first = store.readsFromRedis(id); + for (let i = 0; i < 5; i++) { + expect(store.readsFromRedis(id)).toBe(first); + } + } + }); + + it("gives two instances of the same dial the same answer", () => { + // The cohort must not depend on process state, or a redeploy reshuffles every in-flight run. + const first = probe("redis-read", 50); + const second = probe("redis-read", 50); + + for (const id of ids.slice(0, 50)) { + expect(second.readsFromRedis(id)).toBe(first.readsFromRedis(id)); + } + }); + + it("spreads a population across the dial", () => { + const store = probe("redis-read", 50); + const enabled = ids.filter((id) => store.readsFromRedis(id)).length; + + // A wide band: this asserts the hash spreads at all, not that it is uniform. + expect(enabled).toBeGreaterThan(150); + expect(enabled).toBeLessThan(350); + }); + + it("grows the cohort monotonically as the dial rises", () => { + const at = (percent: number) => { + const store = probe("redis-read", percent); + return new Set(ids.filter((id) => store.readsFromRedis(id))); + }; + + const ten = at(10); + const fifty = at(50); + const ninety = at(90); + + // Raising the dial must only ever add runs. A run that fell out on the way up would flip back to + // Postgres mid-flight, which is the thing the stable hash exists to prevent. + expect([...ten].every((id) => fifty.has(id))).toBe(true); + expect([...fifty].every((id) => ninety.has(id))).toBe(true); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts new file mode 100644 index 00000000000..42924e34c34 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts @@ -0,0 +1,397 @@ +// Reads served from Redis must be indistinguishable from the Postgres reads they replace: the same +// payload shape, the same tenant boundary, the same fallback when Redis does not hold the answer. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + opts?: { mode?: SnapshotStoreMode; readPercent?: number } +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: { method: string; source: string }[] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "redis-read", + readPercent: opts?.readPercent ?? 100, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { decorated, redis, reads }; +} + +async function seedRun( + decorated: TaskRunExecutionSnapshotStore, + env: SnapshotFixtureEnv +): Promise { + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + return runId; +} + +function snapshotInput(runId: string, env: SnapshotFixtureEnv, description: string) { + return { + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("snapshot reads", () => { + containerTest("serves the latest snapshot from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(latest!.id).toBe(created.id); + expect(latest!.executionStatus).toBe("EXECUTING"); + expect(latest!.description).toBe("Run started"); + expect(latest!.runId).toBe(runId); + expect(latest!.checkpoint).toBeNull(); + expect(latest!.completedWaitpoints).toEqual([]); + expect(reads).toContainEqual({ method: "findLatestExecutionSnapshot", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + containerTest("returns the same payload Postgres would", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + expect(fromRedis!.id).toBe(fromPostgres!.id); + expect(fromRedis!.executionStatus).toBe(fromPostgres!.executionStatus); + expect(fromRedis!.description).toBe(fromPostgres!.description); + expect(fromRedis!.runStatus).toBe(fromPostgres!.runStatus); + expect(fromRedis!.attemptNumber).toBe(fromPostgres!.attemptNumber); + expect(fromRedis!.isValid).toBe(fromPostgres!.isValid); + expect(fromRedis!.environmentId).toBe(fromPostgres!.environmentId); + expect(fromRedis!.createdAt.toISOString()).toBe(fromPostgres!.createdAt.toISOString()); + } finally { + await redis.quit(); + } + }); + + containerTest( + "reads a foreign environment as not found, so the caller's 404 still fires", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const foreign = await decorated.findLatestExecutionSnapshot(runId, undefined, "env_other"); + + expect(foreign).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "falls back to Postgres for a run with no keyspace", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A pre-cutover run: it exists in Postgres and Redis has never seen it. + await postgresOnly.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(latest!.executionStatus).toBe("RUN_CREATED"); + expect(reads).toContainEqual({ method: "findLatestExecutionSnapshot", source: "postgres" }); + } finally { + await redis.quit(); + } + } + ); + + containerTest("reads from Postgres at readPercent 0", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + readPercent: 0, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); + + containerTest("reads from Postgres at mode dual-write", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + mode: "dual-write", + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); + + containerTest("serves the since-cursor lookup from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const cursor = await decorated.findExecutionSnapshot({ + where: { id: created.id, runId }, + select: { createdAt: true }, + }); + + expect(cursor).not.toBeNull(); + expect((cursor as { createdAt: Date }).createdAt.toISOString()).toBe( + created.createdAt.toISOString() + ); + expect(reads).toContainEqual({ method: "findExecutionSnapshot", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "delegates a snapshot lookup it does not recognise", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + // A different selection: Redis must not answer it approximately. + const row = await decorated.findExecutionSnapshot({ + where: { id: created.id }, + select: { description: true }, + }); + + expect(row).toEqual({ description: "Run started" }); + expect(reads.filter((r) => r.method === "findExecutionSnapshot")).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest("serves the since window from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const first = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "First")); + await new Promise((resolve) => setTimeout(resolve, 5)); + const second = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Second")); + await new Promise((resolve) => setTimeout(resolve, 5)); + const third = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Third")); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + // Descending, exactly as the engine asked; it reverses app-side. + expect(window.map((s) => s.id)).toEqual([third.id, second.id]); + expect(reads).toContainEqual({ method: "findManyExecutionSnapshots", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "delegates a window query it does not recognise", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "First")); + + const rows = await decorated.findManyExecutionSnapshots({ + where: { runId }, + orderBy: { createdAt: "asc" }, + }); + + expect(rows.length).toBeGreaterThan(0); + expect(reads.filter((r) => r.method === "findManyExecutionSnapshots")).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "serves the waitpoint id projections from Redis", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const ids = await decorated.findSnapshotCompletedWaitpointIds(created.id, undefined, runId); + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + + expect(ids).toEqual([]); + // present distinguishes "no waitpoints" from "this reader cannot see the snapshot", which is + // what the engine's read-repair keys off. + expect(withPresence).toEqual({ present: true, ids: [] }); + expect(reads).toContainEqual({ + method: "findSnapshotCompletedWaitpointIds", + source: "redis", + }); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "delegates a waitpoint id projection with no run id", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + // Without a run id there is no keyspace to look in. + const ids = await decorated.findSnapshotCompletedWaitpointIds(created.id); + + expect(ids).toEqual([]); + expect(reads.filter((r) => r.method.startsWith("findSnapshot"))).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest("never touches Redis for reads at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + mode: "off", + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(await redis.getLatest(runId)).toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts new file mode 100644 index 00000000000..3b75ca4ca05 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts @@ -0,0 +1,231 @@ +// Inside a transaction the Redis append cannot run until the Postgres side commits, or a rollback +// leaves Redis holding a transition that never happened. These tests observe the buffer from inside +// the callback, so the deferral is proved rather than assumed. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build(prisma: never, redisOptions: never, mode: "off" | "dual-write" = "dual-write") { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { store: redis, mode } + ); + return { decorated, redis }; +} + +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function snapshotInput(runId: string, env: SnapshotFixtureEnv, id: string, description: string) { + return { + id, + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("the staging facade", () => { + containerTest("flushes the append after the commit", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, id, "Run started"), tx); + + // Still inside the transaction: nothing has reached Redis yet. + expect(await redis.getById(runId, id)).toBeNull(); + }); + + expect(await redis.getById(runId, id)).not.toBeNull(); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + containerTest( + "writes nothing to Redis when the transaction rolls back", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + await expect( + decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, id, "Run started"), tx); + throw new Error("rolled back"); + }) + ).rejects.toThrow("rolled back"); + + // Both sides agree that the transition never happened. + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id } })).toBe(0); + expect(await redis.getById(runId, id)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest("flushes several staged appends in order", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const first = generateInternalId(); + const second = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, first, "First"), tx); + await store.createExecutionSnapshot(snapshotInput(runId, env, second, "Second"), tx); + }); + + const firstRead = await redis.getById(runId, first); + const secondRead = await redis.getById(runId, second); + expect(firstRead).not.toBeNull(); + expect(secondRead).not.toBeNull(); + // Order matters: the log is append-only and its seq is what orders a read. + expect(firstRead!.seq).toBeLessThan(secondRead!.seq); + } finally { + await redis.quit(); + } + }); + + containerTest( + "hands the transaction callback a decorated store", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + let seen: unknown; + + await decorated.runInTransaction(runId, async (store) => { + seen = store; + }); + + expect(seen).toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect((seen as TaskRunExecutionSnapshotStore).mode).toBe("dual-write"); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "hands the transaction callback the plain delegate at mode off", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + let seen: unknown; + + await decorated.runInTransaction(runId, async (store) => { + seen = store; + }); + + expect(seen).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "wraps the store handle from forWaitpointCompletion", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const handle = await decorated.forWaitpointCompletion(generateInternalId(), { + routeKind: "MANUAL", + } as never); + + // No snapshot write goes through this handle today. Wrapping it is what stops a future one + // from bypassing the decorator with no signal. + expect(handle).toBeInstanceOf(TaskRunExecutionSnapshotStore); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "returns the plain handle from forWaitpointCompletion at mode off", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); + try { + const handle = await decorated.forWaitpointCompletion(generateInternalId(), { + routeKind: "MANUAL", + } as never); + + expect(handle).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts new file mode 100644 index 00000000000..e6468c4f09f --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts @@ -0,0 +1,454 @@ +// A transition writes Postgres first and Redis second. The order is proved by observation, not by +// reading the code: with the Redis half made to fail, the Postgres row is still there and the caller +// sees no error, which is only possible if Postgres went first. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +type Harness = { + decorated: TaskRunExecutionSnapshotStore; + redis: RedisSnapshotStore; + repairs: { runId: string; snapshotId: string; executionStatus: string }[]; + writes: { site: string; outcome: string }[]; +}; + +function harness( + prisma: never, + redisOptions: never, + opts?: { + mode?: SnapshotStoreMode; + faults?: ConstructorParameters[1]["faults"]; + } +): Harness { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const repairs: Harness["repairs"] = []; + const writes: Harness["writes"] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "dual-write", + ...(opts?.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + }, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: () => {}, + }, + } + ); + + return { decorated, redis, repairs, writes }; +} + +/** + * Creates the run and its keyspace, so a following transition is not skippedNoKeyspace. + * + * The birth is appended through the raw store rather than the decorator, because the decorator's + * own birth path is a separate concern with its own suite. Keeping it out here means a failure in + * this file is a failure of the transition path and nothing else. + */ +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function completionInput(env: SnapshotFixtureEnv) { + return { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +function expireInput(env: SnapshotFixtureEnv) { + return { + error: { type: "STRING_ERROR" as const, raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +describe("transition write ordering", () => { + containerTest("writes Postgres then Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.completeAttemptSuccess(runId, completionInput(env), { select: { id: true } }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + const read = await redis.getById(runId, row.id); + + expect(read).not.toBeNull(); + expect(read!.entry.id).toBe(row.id); + expect(read!.entry.executionStatus).toBe("FINISHED"); + expect(writes).toContainEqual({ site: "completeAttemptSuccess", outcome: "written" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "keeps the Postgres write and enqueues one repair when the append fails", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs } = harness(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterPgBeforeRedis") throw new InjectedSnapshotFault(boundary); + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + // The caller must NOT see an error: the Postgres mutation already committed, and the stall + // watchdog is the designed compensator. + await decorated.completeAttemptSuccess(runId, completionInput(env), { + select: { id: true }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + + expect(await redis.getById(runId, row.id)).toBeNull(); + expect(repairs).toEqual([{ runId, snapshotId: row.id, executionStatus: "FINISHED" }]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "treats a transition on a run with no keyspace as skipped, not failed", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + // No birth: this is every pre-cutover run's first transition after the dial moves. + const { run, env } = await setupSnapshotIdFixture(prisma); + + await decorated.expireRun(run.id, expireInput(env), { select: { id: true } }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + expect(await redis.getLatest(run.id)).toBeNull(); + expect(repairs).toEqual([]); + expect(writes).toEqual([{ site: "expireRun", outcome: "skippedNoKeyspace" }]); + } finally { + await redis.quit(); + } + } + ); + + containerTest("appends for expireRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.expireRun(runId, expireInput(env), { select: { id: true } }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + const read = await redis.getById(runId, row.id); + expect(read?.entry.description).toBe("Run expired"); + } finally { + await redis.quit(); + } + }); + + containerTest("appends for expireParkedRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + await prisma.taskRun.update({ where: { id: runId }, data: { status: "PENDING_VERSION" } }); + + const result = await decorated.expireParkedRun(runId, { + ...expireInput(env), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { ...expireInput(env).snapshot, description: "Parked run expired" }, + }); + + expect(result.count).toBe(1); + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + expect((await redis.getById(runId, row.id))?.entry.description).toBe("Parked run expired"); + } finally { + await redis.quit(); + } + }); + + containerTest( + "appends nothing when expireParkedRun matches no run", + async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + // The run is PENDING, so the delegate's `status: PENDING_VERSION` guard matches nothing. + + const result = await decorated.expireParkedRun(runId, { + ...expireInput(env), + statusReason: "VERSION_NEVER_ARRIVED", + }); + + expect(result.count).toBe(0); + expect(writes.filter((w) => w.site === "expireParkedRun")).toEqual([]); + const latest = await redis.getLatest(runId); + expect(latest?.entry.executionStatus).toBe("RUN_CREATED"); + } finally { + await redis.quit(); + } + } + ); + + containerTest("appends for rescheduleRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.rescheduleRun(runId, { + delayUntil: new Date(Date.now() + 60_000), + snapshot: { + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "DELAYED" }, + }); + expect((await redis.getById(runId, row.id))?.entry.description).toBe( + "Delayed run was rescheduled to a future date" + ); + } finally { + await redis.quit(); + } + }); + + containerTest( + "appends nothing when rescheduleRun carries no snapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.rescheduleRun(runId, { delayUntil: new Date(Date.now() + 60_000) }); + + expect(writes.filter((w) => w.site === "rescheduleRun")).toEqual([]); + expect((await redis.getLatest(runId))?.entry.executionStatus).toBe("RUN_CREATED"); + } finally { + await redis.quit(); + } + } + ); + + containerTest("appends for lockRunToWorker under a CAS", async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + const read = await redis.getById(runId, snapshotId); + expect(read?.entry.executionStatus).toBe("PENDING_EXECUTING"); + expect(read?.entry.previousSnapshotId).toBe(head!.id); + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "written" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "reports a forked append without enqueuing a repair", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + // A stale previousSnapshotId: another writer advanced the head. A repair cannot help, so the + // outcome is counted and dropped. + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: generateInternalId(), + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "forked" }); + expect(repairs).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "appends for the standalone createExecutionSnapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const created = await decorated.createExecutionSnapshot({ + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const read = await redis.getById(runId, created.id); + expect(read).not.toBeNull(); + expect(read!.entry.executionStatus).toBe("EXECUTING"); + // The standalone path is the one whose delegate returns the row, so both stores agree exactly. + expect(read!.entry.createdAt).toBe(created.createdAt.toISOString()); + } finally { + await redis.quit(); + } + } + ); + + containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never, { mode: "off" }); + try { + const { run, env } = await setupSnapshotIdFixture(prisma); + + await decorated.completeAttemptSuccess(run.id, completionInput(env), { + select: { id: true }, + }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + expect(await redis.getLatest(run.id)).toBeNull(); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts new file mode 100644 index 00000000000..46a57c6b3bf --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -0,0 +1,891 @@ +// Decorates any RunStore so execution snapshots also land in Redis. It overrides only the methods +// that touch a snapshot and inherits the rest from the generated pass-through base. +// +// Write ORDER is the correctness property, and the two orders are deliberately different: +// +// transition Postgres first, Redis second. A crash in the gap leaves a run whose latest snapshot +// is stale, which is exactly the state the heartbeat stall watchdog already heals. +// birth Redis first, Postgres second. A crash in the gap leaves an unreachable key for a run +// that does not exist. Postgres-first would leave a run with no snapshot at all, and +// getLatestExecutionSnapshot treats that as a hard error. +// +// Each order is chosen so the crash state is the harmless one. A lost cross-store write is never +// recovered by a transaction or an outbox: recovery is always the existing stall-and-repair job. +import { Logger } from "@trigger.dev/core/logger"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { DelegatingRunStore } from "./delegatingRunStore.js"; +import type { + CompletedWaitpointRef, + RedisSnapshotStore, + SnapshotEntryInput, + SnapshotRead, +} from "./redisSnapshotStore.js"; +import { deriveOrder } from "./redisSnapshotStore.js"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, + isTerminalEntry, +} from "./snapshotEntry.js"; +import { isInjectedFault, type SnapshotFaultInjector } from "./snapshotFaultInjection.js"; +import type { + ReadClient, + CompletionSnapshotInput, + CreateCancelledRunInput, + CreateExecutionSnapshotInput, + CreateRunInput, + ExpireSnapshotInput, + LockRunData, + RescheduleSnapshotInput, + RunStore, + TaskRunWithWaitpoint, +} from "./types.js"; +import { matchSinceCursorLookup, matchSinceWindow } from "./snapshotReadShapes.js"; +import { boundedIn } from "@trigger.dev/database"; +import type { Prisma, PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database"; + +/** One initial attempt plus three retries, per the write protocol. */ +const APPEND_ATTEMPTS = 4; + +/** + * Matches the engine's own chunked waitpoint fetch. A batch can complete a thousand waitpoints at + * once, and an unbounded `in:` makes each distinct list length its own prepared statement. + */ +const WAITPOINT_CHUNK_SIZE = 100; + +/** + * The rollout dial. Postgres stays fully written and authoritative in every position before + * `redis-only`, so every earlier position rolls back losslessly by turning the dial down. + * + * `compare` writes exactly as `dual-write` does. Its sampled dual-read and diff are a later ticket; + * the position is named here so the dial does not have to widen once that lands. + */ +export type SnapshotStoreMode = "off" | "dual-write" | "compare" | "redis-read" | "redis-only"; + +/** + * Enqueues the existing `repairSnapshot` job for a run whose append was lost. The decorator lives in + * run-store and cannot reach the engine's worker, so the binding is injected. That binding must + * reuse the stall watchdog's job id for the run, or the watchdog and this path can start two + * concurrent repairs on one run. + */ +export type SnapshotRepairEnqueuer = (args: { + runId: string; + snapshotId: string; + executionStatus: string; +}) => Promise; + +export type DecoratorMetrics = { + recordWrite(site: string, outcome: string): void; + recordAppendFailed(site: string): void; + recordRead(method: string, source: "redis" | "postgres"): void; +}; + +export type TaskRunExecutionSnapshotStoreOptions = { + store: RedisSnapshotStore; + /** Defaults to `off`, which is a pure pass-through that never touches Redis. */ + mode?: SnapshotStoreMode; + /** Percentage of runs whose reads come from Redis at `redis-read` and `redis-only`. Defaults to 0. */ + readPercent?: number; + onAppendFailure?: SnapshotRepairEnqueuer; + faults?: SnapshotFaultInjector; + metrics?: DecoratorMetrics; + logger?: Logger; + /** + * Internal. Set only by the staging facade this class builds for `runInTransaction`. When present, + * an intercepted write does its Postgres half and pushes its entry here instead of appending, and + * the outer instance flushes the buffer after the transaction commits. + */ + staging?: StagedAppend[]; +}; + +/** One deferred append: the entry, plus the wait cycle it carries, if any. */ +export type StagedAppend = { + entry: SnapshotEntryInput; + completedWaitpoints?: CompletedWaitpointRef[]; +}; + +export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { + readonly mode: SnapshotStoreMode; + protected readonly redis: RedisSnapshotStore; + protected readonly readPercent: number; + protected readonly onAppendFailure?: SnapshotRepairEnqueuer; + protected readonly faults?: SnapshotFaultInjector; + protected readonly metrics?: DecoratorMetrics; + protected readonly logger: Logger; + protected readonly staging?: StagedAppend[]; + + constructor(delegate: RunStore, options: TaskRunExecutionSnapshotStoreOptions) { + super(delegate); + this.redis = options.store; + this.mode = options.mode ?? "off"; + this.readPercent = options.readPercent ?? 0; + this.onAppendFailure = options.onAppendFailure; + this.faults = options.faults; + this.metrics = options.metrics; + this.logger = options.logger ?? new Logger("TaskRunExecutionSnapshotStore", "debug"); + this.staging = options.staging; + } + + /** True in every position that appends to Redis. */ + protected get writesRedis(): boolean { + return this.mode !== "off"; + } + + /** + * The staging facade. Two writes share one Postgres transaction here, and the Redis half of each + * cannot run until that transaction commits: a rollback would otherwise leave Redis holding a + * transition that never happened. + * + * The callback gets a second decorator over the transaction-bound store, carrying a staging + * buffer. An intercepted write does its Postgres half through that store and pushes its entry + * onto the buffer. After the transaction resolves, this instance flushes the buffer in order + * through the same retry-and-repair path a lone transition uses. If the callback throws, the + * delegate rejects, the flush never runs, and the buffer goes away with the stack — so the + * Postgres rollback and the Redis silence agree. + */ + override async runInTransaction( + runId: string | undefined, + fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise + ): Promise { + if (!this.writesRedis) { + // At `off` the callback must receive the delegate's own store, untouched, so a transaction + // behaves exactly as it does without the decorator in the chain. + return this.delegate.runInTransaction(runId, fn); + } + + const staged: StagedAppend[] = []; + + const result = await this.delegate.runInTransaction(runId, (store, tx) => + fn(this.#wrap(store, staged), tx) + ); + + // The transaction committed. Only now can a snapshot claim its partner is durable. + for (const item of staged) { + await this.#appendTransition( + "runInTransaction", + item.entry, + undefined, + item.completedWaitpoints + ); + } + + return result; + } + + /** + * `forWaitpointCompletion` hands the caller a store to apply a completion on. No snapshot write + * goes through that handle today, so wrapping it changes nothing now; leaving it unwrapped is the + * one hole that would let a future snapshot write bypass the decorator with no signal at all. + */ + override async forWaitpointCompletion( + waitpointId: string, + context: Parameters[1] + ): Promise { + const store = await this.delegate.forWaitpointCompletion(waitpointId, context); + + if (!this.writesRedis) { + return store; + } + + return this.#wrap(store); + } + + /** + * A second decorator over another store, sharing this one's options. One class in both roles keeps + * the write-ordering logic in exactly one place. Passing no buffer gives a plain decorator that + * appends immediately; passing one makes it stage instead. + */ + #wrap(store: RunStore, staging?: StagedAppend[]): TaskRunExecutionSnapshotStore { + return new TaskRunExecutionSnapshotStore(store, { + store: this.redis, + mode: this.mode, + readPercent: this.readPercent, + logger: this.logger, + ...(this.onAppendFailure && { onAppendFailure: this.onAppendFailure }), + ...(this.faults && { faults: this.faults }), + ...(this.metrics && { metrics: this.metrics }), + ...(staging && { staging }), + }); + } + + // --------------------------------------------------------------------------------------------- + // Births: Redis first, Postgres second. + // --------------------------------------------------------------------------------------------- + + override async createRun( + params: CreateRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + if (!this.writesRedis) { + return this.delegate.createRun(params, tx); + } + + const ctx = this.#context(params.data.id, params.snapshot.id); + const snapshot = { ...params.snapshot, id: ctx.id, createdAt: ctx.createdAt }; + + await this.#appendBirth("createRun", entryFromCreateRun(ctx, snapshot)); + + return this.delegate.createRun({ ...params, snapshot }, tx); + } + + override async createCancelledRun( + params: CreateCancelledRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + if (!this.writesRedis) { + return this.delegate.createCancelledRun(params, tx); + } + + const ctx = this.#context(params.data.id, params.snapshot.id); + const snapshot = { ...params.snapshot, id: ctx.id, createdAt: ctx.createdAt }; + + await this.#appendBirth("createCancelledRun", entryFromCreateRun(ctx, snapshot)); + + return this.delegate.createCancelledRun({ ...params, snapshot }, tx); + } + + // --------------------------------------------------------------------------------------------- + // Transitions: Postgres first, Redis second. + // --------------------------------------------------------------------------------------------- + + override async completeAttemptSuccess( + runId: string, + data: { + completedAt: Date; + output?: string; + outputType: string; + usageDurationMs: number; + costInCents: number; + snapshot: CompletionSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.completeAttemptSuccess(runId, data, args, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.completeAttemptSuccess(runId, withId, args, tx); + + await this.#appendTransition( + "completeAttemptSuccess", + entryFromCompletion(ctx, withId.snapshot) + ); + return result; + } + + override async expireRun( + runId: string, + data: { error: unknown; completedAt: Date; expiredAt: Date; snapshot: ExpireSnapshotInput }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.expireRun(runId, data as never, args, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.expireRun(runId, withId as never, args, tx); + + await this.#appendTransition("expireRun", entryFromExpire(ctx, withId.snapshot)); + return result; + } + + override async expireParkedRun( + runId: string, + data: { + error: unknown; + completedAt: Date; + expiredAt: Date; + statusReason: string; + snapshot: ExpireSnapshotInput; + }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + if (!this.writesRedis) { + return this.delegate.expireParkedRun(runId, data as never, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.expireParkedRun(runId, withId as never, tx); + + // The delegate writes nothing when the run is no longer PENDING_VERSION, so neither does Redis. + if (result.count > 0) { + await this.#appendTransition("expireParkedRun", entryFromExpire(ctx, withId.snapshot)); + } + return result; + } + + override async rescheduleRun( + runId: string, + data: { delayUntil: Date; queueTimestamp?: Date; snapshot?: RescheduleSnapshotInput }, + tx?: PrismaClientOrTransaction + ): Promise { + // The delegate writes a snapshot only when one is supplied, so an absent snapshot is a plain run + // update with nothing for Redis to mirror. + if (!this.writesRedis || !data.snapshot) { + return this.delegate.rescheduleRun(runId, data, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.rescheduleRun(runId, withId, tx); + + await this.#appendTransition("rescheduleRun", entryFromReschedule(ctx, withId.snapshot)); + return result; + } + + override async lockRunToWorker( + runId: string, + data: LockRunData, + tx?: PrismaClientOrTransaction + ): Promise>> { + if (!this.writesRedis) { + return this.delegate.lockRunToWorker(runId, data, tx); + } + + // This is the one transition whose input already carries both an id and the previous snapshot + // id, so it is also the one that can append under a compare-and-set on the current head. + const ctx = { id: data.snapshot.id, runId, createdAt: new Date() }; + const withStamp = { ...data, snapshot: { ...data.snapshot, createdAt: ctx.createdAt } }; + + const result = await this.delegate.lockRunToWorker(runId, withStamp, tx); + + await this.#appendTransition( + "lockRunToWorker", + entryFromLock(ctx, withStamp.snapshot), + withStamp.snapshot.previousSnapshotId, + // The lock site already carries the resolved order, so the refs are rebuilt from it rather + // than re-derived: its index IS the position in that list. + withStamp.snapshot.completedWaitpointOrder.map((id, index) => ({ id, index })) + ); + return result; + } + + override async createExecutionSnapshot( + input: CreateExecutionSnapshotInput, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.createExecutionSnapshot(input, tx); + } + + const ctx = this.#context(input.run.id, input.id); + const created = await this.delegate.createExecutionSnapshot( + { ...input, id: ctx.id, createdAt: ctx.createdAt }, + tx + ); + + // The standalone path is the only one whose delegate returns the row, so its entry can take the + // exact createdAt Postgres recorded rather than the decorator's own clock. + await this.#appendTransition( + "createExecutionSnapshot", + entryFromCreateExecutionSnapshot(ctx, input), + input.previousSnapshotId, + input.completedWaitpoints + ); + return created; + } + + // --------------------------------------------------------------------------------------------- + // The append protocol. + // --------------------------------------------------------------------------------------------- + + /** Mints the id when the caller did not, and stamps one clock for both stores. */ + #context(runId: string, suppliedId?: string) { + return { id: suppliedId ?? generateInternalId(), runId, createdAt: new Date() }; + } + + /** + * Births invert the order. Postgres-first would leave a run with no snapshot at all, and + * `getLatestExecutionSnapshot` treats that as a hard error, so the run would be stuck. Redis-first + * leaves an orphaned keyspace for a run that does not exist, which nothing can reach and the + * sweep's second rule reaps. + * + * Being first is also what lets this path refuse. Before `redis-only` a failed birth append is + * survivable, because Postgres is authoritative and holds the snapshot; at `redis-only` Postgres + * writes no snapshot, so a run created without its Redis birth would have no snapshot anywhere. + * Throwing here happens before the run row exists, so the caller retries a clean creation. + */ + async #appendBirth(site: string, entry: SnapshotEntryInput): Promise { + if (this.staging) { + // A birth inside a transaction cannot be staged: staging flushes after the commit, which is + // the opposite of what a birth needs. No caller does this today, so say so and append now. + this.logger.error("a run birth inside a transaction cannot be staged", { + runId: entry.runId, + site, + }); + } + + for (let attempt = 0; attempt < APPEND_ATTEMPTS; attempt++) { + try { + const result = await this.redis.append({ + entry, + kind: "birth", + isTerminal: isTerminalEntry(entry), + }); + this.#recordOutcome(site, entry, result); + + // Modelled AFTER the successful append: the crash this boundary represents is a process that + // died between the two stores, not an append that failed. + this.faults?.("afterRedisBirthBeforePg", { runId: entry.runId, snapshotId: entry.id }); + return; + } catch (error) { + if (isInjectedFault(error)) { + throw error; + } + + if (attempt === APPEND_ATTEMPTS - 1) { + this.metrics?.recordAppendFailed(site); + this.logger.error("snapshot birth append failed after retries", { + runId: entry.runId, + snapshotId: entry.id, + site, + mode: this.mode, + error, + }); + + if (this.mode === "redis-only") { + throw error; + } + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt)); + } + } + } + + /** + * Postgres has already committed by the time this runs. A throw here would turn a gap the stall + * watchdog heals into a caller-visible failure, so it never rethrows: it retries, then hands the + * run to the repair job and returns. + */ + async #appendTransition( + site: string, + entry: SnapshotEntryInput, + expectedCur?: string, + completedWaitpoints?: CompletedWaitpointRef[] + ): Promise { + if (this.staging) { + // Inside a transaction the append cannot run until the Postgres side commits, or a rollback + // leaves Redis holding a transition that never happened. + this.staging.push({ entry, ...(completedWaitpoints && { completedWaitpoints }) }); + return; + } + + for (let attempt = 0; attempt < APPEND_ATTEMPTS; attempt++) { + try { + this.faults?.(attempt === 0 ? "afterPgBeforeRedis" : "midFlushRetry", { + runId: entry.runId, + snapshotId: entry.id, + }); + + const cycle = await this.#resolveCycle(entry.runId, completedWaitpoints); + + const result = await this.redis.append({ + entry, + kind: "transition", + isTerminal: isTerminalEntry(entry), + ...(expectedCur !== undefined && { expectedCur }), + ...(cycle && { cycle }), + }); + + this.#recordOutcome(site, entry, result); + return; + } catch (error) { + // An injected fault models a dead process, not a retryable append failure. + if (isInjectedFault(error)) { + this.metrics?.recordAppendFailed(site); + await this.#enqueueRepair(entry); + return; + } + + if (attempt === APPEND_ATTEMPTS - 1) { + this.metrics?.recordAppendFailed(site); + this.logger.error("snapshot append failed after retries", { + runId: entry.runId, + snapshotId: entry.id, + site, + error, + }); + await this.#enqueueRepair(entry); + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt)); + } + } + } + + /** + * Decides whether this append mints a new wait cycle or points at the one already there. + * + * A resume append carries a newly-differing id set, so it mints a cycle and the record set is + * written once. Every copy-forward append that follows re-passes the SAME list, and re-minting on + * each would rewrite the record set once per entry in the resume chain — the write amplification + * the pointer model exists to remove. So an unchanged id set carries the previous cycleSeq + * forward and writes no key. + * + * The extra read only happens for an append that actually carries waitpoints, which is the resume + * path rather than the hot path. + * + * `records` is deliberately left unset. The record envelope belongs to the waitpoint lane and + * ships empty in this build, so dual-write never re-versions the entry when it arrives. + */ + async #resolveCycle( + runId: string, + completedWaitpoints?: CompletedWaitpointRef[] + ): Promise< + | { kind: "new"; completedWaitpoints: CompletedWaitpointRef[] } + | { kind: "carryForward"; cycleSeq: number } + | undefined + > { + if (!completedWaitpoints || completedWaitpoints.length === 0) { + return undefined; + } + + const order = deriveOrder(completedWaitpoints); + + try { + const head = await this.redis.getLatest(runId); + const previous = head?.completedWaitpointIds?.order; + + if (head?.cycle && previous && sameOrder(previous, order)) { + return { kind: "carryForward", cycleSeq: head.cycle.cycleSeq }; + } + } catch (error) { + // A failed probe must not lose the waitpoints. Minting a fresh cycle is the safe direction: + // it costs one duplicated record set, where a wrong carryForward would point at another + // cycle's ids. + this.logger.warn("snapshot cycle probe failed, minting a new cycle", { runId, error }); + } + + return { kind: "new", completedWaitpoints }; + } + + /** + * None of the four append outcomes is a failure, and none of them enqueues a repair. + * + * `skippedNoKeyspace` is every pre-cutover run's transitions. `forked` means another writer + * advanced the head, which a repair cannot help. `duplicate` is a retry that already landed. + * `cycleMismatch` means the store refused an untrustworthy waitpoint pointer on purpose. + */ + #recordOutcome( + site: string, + entry: SnapshotEntryInput, + result: Awaited> + ): void { + this.metrics?.recordWrite(site, result.outcome); + + if (result.outcome === "forked") { + this.logger.warn("snapshot append forked", { + runId: entry.runId, + snapshotId: entry.id, + site, + actualCur: result.actualCur, + }); + } + } + + // --------------------------------------------------------------------------------------------- + // Reads. + // + // Only three production call sites exist, all in the engine's executionSnapshotSystem, all with + // fixed argument shapes. The two generic Prisma-args methods therefore recognise exactly the + // shapes the engine sends and delegate everything else: an unrecognised shape must go to Postgres, + // never get an approximate answer from Redis. + // --------------------------------------------------------------------------------------------- + + /** + * Whether this run's reads come from Redis. Hashed on the run id so a run does not change store + * between two reads of the same poll, which would let a caller see the log go backwards. + */ + protected readsFromRedis(runId: string): boolean { + if (this.mode !== "redis-read" && this.mode !== "redis-only") return false; + if (this.readPercent >= 100) return true; + if (this.readPercent <= 0) return false; + + let hash = 0; + for (let i = 0; i < runId.length; i++) { + hash = (hash * 31 + runId.charCodeAt(i)) >>> 0; + } + return hash % 100 < this.readPercent; + } + + override async findLatestExecutionSnapshot( + runId: string, + client?: ReadClient, + environmentId?: string + ): Promise | null> { + if (!this.readsFromRedis(runId)) { + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + const read = await this.redis.getLatest(runId, { ...(environmentId && { environmentId }) }); + if (!read) { + // A miss is the coexistence path: a pre-cutover run, or expired history. It is not an error. + this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + this.metrics?.recordRead("findLatestExecutionSnapshot", "redis"); + return this.#hydrate(read, runId, client, { hydrateWaitpointRows: true }); + } + + override async findExecutionSnapshot( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + const shape = matchSinceCursorLookup(args); + if (!shape || !this.readsFromRedis(shape.runId)) { + return this.delegate.findExecutionSnapshot(args, client); + } + + const found = await this.redis.getById(shape.runId, shape.id, { + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + + if (!found) { + this.metrics?.recordRead("findExecutionSnapshot", "postgres"); + return this.delegate.findExecutionSnapshot(args, client); + } + + this.metrics?.recordRead("findExecutionSnapshot", "redis"); + // The engine selects createdAt only, so the answer is the cursor and nothing else. + return { + createdAt: new Date(found.entry.createdAt as string), + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload; + } + + override async findManyExecutionSnapshots( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + const shape = matchSinceWindow(args); + if (!shape || !this.readsFromRedis(shape.runId)) { + return this.delegate.findManyExecutionSnapshots(args, client); + } + + const result = await this.redis.getSinceCreatedAt(shape.runId, shape.createdAt, { + limit: shape.take, + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + + if (result.kind === "miss") { + this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); + return this.delegate.findManyExecutionSnapshots(args, client); + } + + this.metrics?.recordRead("findManyExecutionSnapshots", "redis"); + + // The engine asks for createdAt DESC and reverses app-side; the store returns ascending. + const descending = [...result.entries].reverse(); + // Rows are hydrated for no entry here: the engine fetches the head's waitpoints itself, from + // the ids this call's head row reports. Each row still carries its own order. + const hydrated = await Promise.all( + descending.map((entry) => this.#hydrate(entry, shape.runId, client)) + ); + return hydrated as unknown as Prisma.TaskRunExecutionSnapshotGetPayload[]; + } + + override async findSnapshotCompletedWaitpointIds( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise { + // Without a run id there is no keyspace to look in, so the router's fan-out is the only answer. + if (!runId || !this.readsFromRedis(runId)) { + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + if (!ids.present) { + this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "postgres"); + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "redis"); + return ids.distinctIds; + } + + override async findSnapshotCompletedWaitpointIdsWithPresence( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise<{ present: boolean; ids: string[] }> { + if (!runId || !this.readsFromRedis(runId)) { + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + if (!ids.present) { + // present=false means this reader cannot see the snapshot, so its empty list is not + // authoritative and the engine's read-repair needs the Postgres answer. + this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "postgres"); + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "redis"); + return { present: true, ids: ids.distinctIds }; + } + + /** + * Turns a store entry into the Prisma payload the interface promises. + * + * The entry supplies every scalar column. `checkpoint` and the full waitpoint rows still live in + * Postgres, so they are read back through the delegate — but only when the entry says they exist, + * which keeps the common read (a running run with neither) free of any Postgres call at all. + */ + async #hydrate( + read: SnapshotRead, + runId: string, + client?: ReadClient, + opts?: { hydrateWaitpointRows?: boolean } + ): Promise< + Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { completedWaitpoints: true; checkpoint: true }; + }> + > { + const entry = read.entry as Record; + + const checkpoint = entry.checkpointId + ? await this.#hydrateCheckpoint(runId, read.id, client) + : null; + + // `completedWaitpointOrder` is a scalar column, NOT the join. The engine reads it off the head + // row as the index oracle that gives each completed waitpoint its position in a batch, so it + // must be populated even when the waitpoint ROWS are not fetched. Returning an empty order here + // resumes every batched triggerAndWait with `index: undefined`. + const ids = + read.completedWaitpointIds ?? (await this.redis.getSnapshotWaitpointIds(runId, read.id)); + const completedWaitpointOrder = ids.order; + + // The rows themselves are head-only, mirroring the engine's own N x M avoidance. + const completedWaitpoints = opts?.hydrateWaitpointRows + ? await this.#fetchWaitpointsInChunks(ids.distinctIds, runId, client) + : []; + + return { + id: read.id, + engine: entry.engine ?? "V2", + executionStatus: entry.executionStatus, + description: entry.description, + previousSnapshotId: entry.previousSnapshotId ?? null, + runId: entry.runId, + runStatus: entry.runStatus, + attemptNumber: entry.attemptNumber ?? null, + batchId: entry.batchId ?? null, + environmentId: entry.environmentId, + environmentType: entry.environmentType, + projectId: entry.projectId, + organizationId: entry.organizationId, + checkpointId: entry.checkpointId ?? null, + workerId: entry.workerId ?? null, + runnerId: entry.runnerId ?? null, + metadata: entry.metadata ?? null, + completedWaitpointOrder, + isValid: read.isValid, + error: entry.error ?? null, + createdAt: new Date(entry.createdAt as string), + // A snapshot row is write-once, so both columns hold the one instant the decorator minted. + updatedAt: new Date(entry.createdAt as string), + checkpoint, + completedWaitpoints, + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { completedWaitpoints: true; checkpoint: true }; + }>; + } + + /** + * Chunked, and bounded within each chunk, mirroring the engine's own waitpoint fetch. The run id + * routes each chunk to the owning store rather than fanning every one across both databases. + */ + async #fetchWaitpointsInChunks( + waitpointIds: string[], + runId: string, + client?: ReadClient + ): Promise { + if (waitpointIds.length === 0) return []; + + const all: unknown[] = []; + for (let i = 0; i < waitpointIds.length; i += WAITPOINT_CHUNK_SIZE) { + const chunk = waitpointIds.slice(i, i + WAITPOINT_CHUNK_SIZE); + const rows = await this.delegate.findManyWaitpoints( + { where: { id: { in: boundedIn(chunk) } } }, + client, + runId + ); + all.push(...rows); + } + return all; + } + + /** + * Reads the checkpoint row through the snapshot the delegate still holds, so the read stays + * residency-aware: the run id in the where is what routes it to the owning database, and the + * decorator sits above the router and has no client of its own. + * + * At `redis-only` the Postgres snapshot row is gone, so this returns null. The checkpoint row + * itself stays in Postgres, but the interface has no residency-aware way to read one directly. + * Closing that needs a narrow lookup on the interface, which the plan freezes for this ticket. + */ + async #hydrateCheckpoint( + runId: string, + snapshotId: string, + client?: ReadClient + ): Promise { + const row = await this.delegate.findExecutionSnapshot( + { where: { id: snapshotId, runId }, include: { checkpoint: true } }, + client + ); + return (row as { checkpoint?: unknown } | null)?.checkpoint ?? null; + } + + async #enqueueRepair(entry: SnapshotEntryInput): Promise { + if (!this.onAppendFailure) { + return; + } + + try { + await this.onAppendFailure({ + runId: entry.runId, + snapshotId: entry.id, + executionStatus: entry.executionStatus, + }); + } catch (error) { + // The repair enqueue is itself best-effort. Failing it must not fail the caller's write. + this.logger.error("snapshot repair enqueue failed", { runId: entry.runId, error }); + } + } +} + +/** Position-sensitive: the same ids in a different order are a different wait cycle. */ +function sameOrder(a: string[], b: string[]): boolean { + return a.length === b.length && a.every((id, index) => id === b[index]); +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts new file mode 100644 index 00000000000..1879fe1246b --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -0,0 +1,363 @@ +// The completed-waitpoint path, which every other suite here was blind to. +// +// Two defects hid behind that blindness. The decorator passed no cycle to `append`, so no +// wp: key was ever written and the Redis waitpoint side was permanently empty. And the +// since-window hydration returned an empty `completedWaitpointOrder`, which is the index oracle the +// engine uses to give each completed waitpoint its position in a batch — an empty order resumes +// every batched triggerAndWait with `index: undefined`. +// +// So these tests all use a snapshot that ACTUALLY carries waitpoints. A test that does not cannot +// tell a working cycle from a missing one. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWaitpoints, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + mode: "dual-write" | "redis-read" = "redis-read" +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const writes: { site: string; outcome: string }[] = []; + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode, + readPercent: 100, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: () => {}, + }, + } + ); + return { decorated, redis, writes }; +} + +async function seedRun( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + env: SnapshotFixtureEnv +): Promise { + const runId = generateInternalId(); + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); + return runId; +} + +function resumeInput( + runId: string, + env: SnapshotFixtureEnv, + completedWaitpoints: { id: string; index?: number }[], + description = "Run resumed" +) { + return { + id: generateInternalId(), + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + completedWaitpoints, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("completed-waitpoint cycles", () => { + containerTest("a resume append mints a cycle key", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + // The key exists at all — before the fix, none was ever written. + const cycleKeys = await probe.keys(`snap:{${runId}}:wp:*`); + expect(cycleKeys.length).toBe(1); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.present).toBe(true); + expect(ids.order).toEqual([wpA, wpB]); + expect(ids.distinctIds).toEqual([wpA, wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "a copy-forward reuses the cycle and writes no second key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const waitpoints = [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]; + + await decorated.createExecutionSnapshot(resumeInput(runId, env, waitpoints, "resume")); + // The same id set again: this is the copy-forward every dequeue and checkpoint site does. + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, waitpoints, "carry one") + ); + const third = await decorated.createExecutionSnapshot( + resumeInput(runId, env, waitpoints, "carry two") + ); + + // Still ONE key. Re-minting per entry is the write amplification the pointer model removes. + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(1); + + // And every entry still resolves the same order. + for (const id of [second.id, third.id]) { + expect((await redis.getSnapshotWaitpointIds(runId, id)).order).toEqual([wpA, wpB]); + } + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a newly-differing id set mints a second cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "first wait") + ); + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpB, index: 0 }], "second wait") + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + expect((await redis.getSnapshotWaitpointIds(runId, second.id)).order).toEqual([wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the same ids in a different order are a new cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + // Order IS the index oracle, so a reordering is a different cycle, not a carry-forward. + const reordered = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpB, index: 0 }, + { id: wpA, index: 1 }, + ]) + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + expect((await redis.getSnapshotWaitpointIds(runId, reordered.id)).order).toEqual([ + wpB, + wpA, + ]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("a repeated id keeps both of its positions", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpX] = await seedSnapshotWaitpoints(prisma, env, 1); + + // One run batched twice under a single idempotency key: the id repeats, and each position + // must survive, because the runner matches results to positions. + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpX, index: 0 }, + { id: wpX, index: 1 }, + ]) + ); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.order).toEqual([wpX, wpX]); + expect(ids.distinctIds).toEqual([wpX]); + } finally { + await redis.quit(); + } + }); + + containerTest( + "findLatestExecutionSnapshot returns the index oracle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + // completedWaitpointOrder is a scalar column, not the join. Empty here means every batched + // waitpoint resumes with index undefined. + expect(latest!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the since-window head carries the index oracle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [], "before the wait") + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + // The head is first in a descending window. This is the row the engine reads the oracle off. + expect(window[0]!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "lockRunToWorker carries its resolved order into the cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: undefined, + lockedToVersionId: undefined, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [wpA, wpB], + completedWaitpointOrder: [wpA, wpB], + }, + } as never); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect(ids.order).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "an append with no waitpoints writes no cycle key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + + await decorated.createExecutionSnapshot(resumeInput(runId, env, [], "no waitpoints")); + + expect(await probe.keys(`snap:{${runId}}:wp:*`)).toEqual([]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); diff --git a/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts new file mode 100644 index 00000000000..e0b8a1a2107 --- /dev/null +++ b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts @@ -0,0 +1,167 @@ +// Shared setup for the snapshot-id, snapshot-writes and entry-parity suites. Modelled on the +// seedEnvironment/buildCreateRunInput pair in PostgresRunStore.test.ts; the slugs are suffixed so +// several fixtures can coexist in one database. +import type { PrismaClient, TaskRunStatus } from "@trigger.dev/database"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import type { CreateRunData } from "../types.js"; + +export type SnapshotFixtureEnv = { + id: string; + type: "DEVELOPMENT"; + projectId: string; + organizationId: string; +}; + +export type SnapshotIdFixture = { + run: { id: string }; + env: SnapshotFixtureEnv; +}; + +export async function seedSnapshotEnvironment(prisma: PrismaClient): Promise { + const suffix = generateInternalId().slice(-12); + + const organization = await prisma.organization.create({ + data: { title: `Snapshot Org ${suffix}`, slug: `snapshot-org-${suffix}` }, + }); + + const project = await prisma.project.create({ + data: { + name: `Snapshot Project ${suffix}`, + slug: `snapshot-project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: `dev-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + + return { + id: environment.id, + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + }; +} + +export function buildCreateRunData(runId: string, env: SnapshotFixtureEnv): CreateRunData { + return { + id: runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${runId.slice(-16)}`, + runtimeEnvironmentId: env.id, + environmentType: env.type, + organizationId: env.organizationId, + projectId: env.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${runId.slice(-8)}`, + spanId: `span_${runId.slice(-8)}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +export type SnapshotWorkerFixture = { workerId: string; taskId: string }; + +/** + * Seeds a BackgroundWorker and one of its tasks. The snapshot's `workerId` and the run's + * `lockedById` are both foreign keys, so a made-up id fails the constraint rather than the + * assertion, and the test reports a fixture fault as if it were a parity fault. + */ +export async function seedSnapshotWorker( + prisma: PrismaClient, + env: SnapshotFixtureEnv +): Promise { + const suffix = generateInternalId().slice(-12); + + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker_${suffix}`, + engine: "V2", + contentHash: `hash_${suffix}`, + projectId: env.projectId, + runtimeEnvironmentId: env.id, + version: "20260824.1", + metadata: {}, + }, + }); + + const task = await prisma.backgroundWorkerTask.create({ + data: { + slug: "my-task", + friendlyId: `task_${suffix}`, + filePath: "src/trigger/my-task.ts", + exportName: "myTask", + workerId: worker.id, + projectId: env.projectId, + runtimeEnvironmentId: env.id, + }, + }); + + return { workerId: worker.id, taskId: task.id }; +} + +/** + * Seeds real Waitpoint rows and returns their ids. The legacy completed-waitpoint join carries a + * real foreign key, so an invented id fails the constraint rather than the assertion — the test + * then reports a fixture fault as if it were a defect in the code under test. + */ +export async function seedSnapshotWaitpoints( + prisma: PrismaClient, + env: SnapshotFixtureEnv, + count: number +): Promise { + const ids: string[] = []; + + for (let i = 0; i < count; i++) { + const suffix = generateInternalId().slice(-12); + const waitpoint = await prisma.waitpoint.create({ + data: { + friendlyId: `waitpoint_${suffix}`, + type: "MANUAL", + status: "COMPLETED", + completedAt: new Date(), + idempotencyKey: `idem_${suffix}`, + userProvidedIdempotencyKey: false, + projectId: env.projectId, + environmentId: env.id, + }, + }); + ids.push(waitpoint.id); + } + + return ids; +} + +/** + * Seeds an environment plus one run in `status`, with no execution snapshot. The suites that use it + * assert on the snapshot rows a store method writes, so the run must start with none. + */ +export async function setupSnapshotIdFixture( + prisma: PrismaClient, + opts?: { status?: TaskRunStatus } +): Promise { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: opts?.status ?? "PENDING" }, + }); + + return { run: { id: runId }, env }; +} diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index 7c7f9566893..93fffe0ee88 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -29,6 +29,11 @@ export type IdempotencyKeyRunMatch = { }; export type CreateRunSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; id?: string; engine: "V2"; executionStatus: TaskRunExecutionStatus; @@ -43,6 +48,14 @@ export type CreateRunSnapshotInput = { }; export type CompletionSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; executionStatus: "FINISHED"; description: string; runStatus: TaskRunStatus; @@ -64,6 +77,14 @@ export type PromotePendingVersionArgs = { }; export type ExpireSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; engine: "V2"; executionStatus: "FINISHED"; description: string; @@ -75,6 +96,14 @@ export type ExpireSnapshotInput = { }; export type RescheduleSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; environmentId: string; environmentType: RuntimeEnvironmentType; projectId: string; @@ -85,6 +114,11 @@ export type RescheduleSnapshotInput = { }; export type LockSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; id: string; previousSnapshotId: string; attemptNumber?: number; @@ -292,6 +326,14 @@ export type TaskRunWithWaitpoint = TaskRun & { associatedWaitpoint: Waitpoint | * input — callers pass the high-level shape, not a raw Prisma `data`/`include`. */ export type CreateExecutionSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; run: { id: string; status: TaskRunStatus; attemptNumber?: number | null }; snapshot: { executionStatus: TaskRunExecutionStatus; diff --git a/knip.json b/knip.json index 84456756ca1..8c32f5e72c3 100644 --- a/knip.json +++ b/knip.json @@ -37,6 +37,9 @@ "internal-packages/observability-map": { "entry": ["src/index.ts", "fixtures/**/*.{js,mjs,cjs,ts,mts,cts,tsx}"] }, + "internal-packages/run-store": { + "entry": ["scripts/**/*.{js,mjs,cjs,ts,mts,cts}"] + }, "internal-packages/otlp-importer": { "ignoreDependencies": ["ts-proto"] },