diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 66be4d132..51b7ec517 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -4,9 +4,10 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Commands -Use Node.js 24 or newer for all commands in this repository. Native dependencies such as -`better-sqlite3` are built against the active Node ABI, so running Vitest or `npx` with Node 20 -can produce misleading SQLite failures. +Use Node.js 24 or newer for all commands in this repository. The SQLite backend uses the +built-in `node:sqlite` module, which is only stable from Node 24 (it is experimental and +flag-gated on older releases), so running Vitest or `npx` with an older Node can produce +misleading SQLite failures. ```sh bun run build # Full build (all packages + integrations + examples, via Turbo) @@ -68,7 +69,7 @@ Each package builds two runtime targets via `bun build --target=X`: Types built with `tsc` (composite + incremental). Conditional exports in `package.json` resolve automatically per runtime. -**No `bun` entry unless it differs.** Bun is not a build target by default: with no `"bun"` condition in `exports`, Bun resolves the default `"import"` and loads the node build. Add a `src/bun.ts`, a `--target=bun` build, and a `"bun"` export condition only when the Bun code genuinely differs — a duplicate of `node.ts` is a third bundle and a third `.d.ts` to keep in sync for no behavior change. Only three entries qualify today: `@workglow/util`'s `"."` (`Worker.bun` vs `Worker.node`), `@workglow/util`'s `"./worker"` (`dist/worker-bun.js` vs `dist/worker-node.js`), and `@workglow/sqlite`'s `./storage` (`bun:sqlite` vs the node driver). That set is pinned by `packages/test/src/test/util/BunExportConditions.test.ts` — adding or removing a `"bun"` condition fails it until the fixture and this paragraph are updated together. +**No `bun` entry unless it differs.** Bun is not a build target by default: with no `"bun"` condition in `exports`, Bun resolves the default `"import"` and loads the node build. Add a `src/bun.ts`, a `--target=bun` build, and a `"bun"` export condition only when the Bun code genuinely differs — a duplicate of `node.ts` is a third bundle and a third `.d.ts` to keep in sync for no behavior change. Only two entries qualify today: `@workglow/util`'s `"."` (`Worker.bun` vs `Worker.node`) and `@workglow/util`'s `"./worker"` (`dist/worker-bun.js` vs `dist/worker-node.js`). `@workglow/sqlite`'s `./storage` used to, but both runtimes now share the `node:sqlite` driver. That set is pinned by `packages/test/src/test/util/BunExportConditions.test.ts` — adding or removing a `"bun"` condition fails it until the fixture and this paragraph are updated together. Exception: vendor packages under `providers/*` (e.g. `@workglow/anthropic`, `@workglow/openai`, `@workglow/google-gemini`) ship `./ai` and `./ai-runtime` sub-paths instead of browser/node. diff --git a/.github/workflows/nightly-typecheck.yml b/.github/workflows/nightly-typecheck.yml index 6b2ffeedb..90a13cc4a 100644 --- a/.github/workflows/nightly-typecheck.yml +++ b/.github/workflows/nightly-typecheck.yml @@ -1,3 +1,5 @@ +# `bun-version: canary` matches `engines.bun` / `packageManager`: no tagged Bun +# release carries `node:sqlite` yet, which @workglow/sqlite/storage requires. name: Nightly type-drift guard permissions: contents: read @@ -19,7 +21,7 @@ jobs: node-version: 24 - uses: oven-sh/setup-bun@v2 with: - bun-version: latest + bun-version: canary - run: bun i - name: Build type declarations run: bun run build:types diff --git a/.github/workflows/publish-preview.yml-off b/.github/workflows/publish-preview.yml-off index 1f4f3e834..3d45fb107 100644 --- a/.github/workflows/publish-preview.yml-off +++ b/.github/workflows/publish-preview.yml-off @@ -19,7 +19,7 @@ jobs: node-version: 24 - uses: oven-sh/setup-bun@v2 with: - bun-version: latest + bun-version: canary - run: bun i - run: bun run build # Publish every workspace under packages/* and providers/* as a PR- diff --git a/bun.lock b/bun.lock index 53baf7f38..e08f4d3c1 100644 --- a/bun.lock +++ b/bun.lock @@ -788,11 +788,9 @@ "devDependencies": { "@sqlite.org/sqlite-wasm": "catalog:", "@sqliteai/sqlite-vector": "catalog:", - "@types/better-sqlite3": "^9.6.0", "@workglow/job-queue": "workspace:*", "@workglow/storage": "workspace:*", "@workglow/util": "workspace:*", - "better-sqlite3": "catalog:", }, "peerDependencies": { "@sqlite.org/sqlite-wasm": "catalog:", @@ -800,12 +798,10 @@ "@workglow/job-queue": "workspace:*", "@workglow/storage": "workspace:*", "@workglow/util": "workspace:*", - "better-sqlite3": "catalog:", }, "optionalPeers": [ "@sqlite.org/sqlite-wasm", "@sqliteai/sqlite-vector", - "better-sqlite3", ], }, "providers/stable-diffusion-server": { @@ -890,7 +886,6 @@ }, "trustedDependencies": [ "protobufjs", - "better-sqlite3", "sharp", "onnxruntime-node", "node-llama-cpp", @@ -922,7 +917,6 @@ "@types/pg": "^8.21.0", "@types/react": "^19.2.18", "aws-sdk-client-mock": "^4.0.0", - "better-sqlite3": "^13.0.3", "commander": "^15.0.0", "electron": "^43.3.0", "fake-indexeddb": "^6.2.4", @@ -1478,8 +1472,6 @@ "@turbo/windows-arm64": ["@turbo/windows-arm64@2.10.10", "", { "os": "win32", "cpu": "arm64" }, "sha512-PMk6zQN0csUFklLe+1hz/5G9uU1YmV0cEIey2R/bSeA6o69qcBTlN4A3jOqkgenOO5dOpMHmq2sEUZo8r1+Ssg=="], - "@types/better-sqlite3": ["@types/better-sqlite3@9.6.0", "", { "dependencies": { "@types/node": "*" } }, "sha512-ZEEwBSgMu7GYJOynoagg5X9JbxfL6dTJDsgViJIqh67jV44kyOr9RXfmFjLK5rzC4MWssP06t9hu/JwGDnUbCg=="], - "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], @@ -1740,8 +1732,6 @@ "baseline-browser-mapping": ["baseline-browser-mapping@2.11.14", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-JyJ954WzuIR8/FFzX0o5krdSTrBAkcCSRfWSleRsIHSWV+cZe2FI1PKggVkFke1hBldRs+LRxUczzE9iPmgZww=="], - "better-sqlite3": ["better-sqlite3@13.0.3", "", { "dependencies": { "node-addon-api": "^8.0.0" } }, "sha512-RbOBxmLBG8uvFUc15X9+9SFemKcQ0WBuISBVkpuiaUB2qblC8UWlHEjdWVoZ8AdhSwmoEgsiXKfopX0CQxaACQ=="], - "bidi-js": ["bidi-js@1.0.3", "", { "dependencies": { "require-from-string": "^2.0.2" } }, "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw=="], "bignumber.js": ["bignumber.js@9.3.1", "", {}, "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ=="], diff --git a/docs/technical/18-multi-runtime-abstraction.md b/docs/technical/18-multi-runtime-abstraction.md index 6eedf532d..210f523a4 100644 --- a/docs/technical/18-multi-runtime-abstraction.md +++ b/docs/technical/18-multi-runtime-abstraction.md @@ -52,8 +52,8 @@ Bun supports every Node.js API the packages here rely on, so a `bun.ts` that is copy of `node.ts` buys nothing but a third build target and a third `.d.ts` to keep in sync. With no `"bun"` condition in `exports`, Bun falls through to the `"import"`/`"types"` default and gets the Node entry — the same code the duplicated file would have given it. Only two entries in the -monorepo earn a Bun build: `@workglow/util` (`Worker.bun` vs `Worker.node`) and -`@workglow/sqlite`'s `./storage` sub-path (`bun:sqlite` vs the Node driver). +monorepo earn a Bun build, both in `@workglow/util`: its `"."` and `"./worker"` sub-paths +(`Worker.bun` vs `Worker.node`). Each platform entry point re-exports everything from `common.ts` and then layers on platform-specific modules. For `@workglow/util`, the entry points look like this: diff --git a/docs/technical/19-build-system.md b/docs/technical/19-build-system.md index a02cda688..9e4759268 100644 --- a/docs/technical/19-build-system.md +++ b/docs/technical/19-build-system.md @@ -232,12 +232,12 @@ Bun is not a third target by default. A package's `exports` carries no `"bun"` c resolves the default `"import"` and loads `dist/node.js` — which is exactly what a `src/bun.ts` identical to `src/node.ts` would have produced. Add a `src/bun.ts`, a `--target=bun` build, and a `"bun"` export condition only when the Bun code genuinely differs; a duplicate is a third bundle -and a third `.d.ts` to keep in sync for no behavior change. Three entries qualify today: -`@workglow/util`'s `"."` (`Worker.bun` vs `Worker.node`), `@workglow/util`'s `"./worker"` -(`dist/worker-bun.js` vs `dist/worker-node.js`), and `@workglow/sqlite`'s `./storage` -(`bun:sqlite` vs the Node driver). That set is pinned by +and a third `.d.ts` to keep in sync for no behavior change. Two entries qualify today, both in +`@workglow/util`: `"."` (`Worker.bun` vs `Worker.node`) and `"./worker"` (`dist/worker-bun.js` +vs `dist/worker-node.js`). `@workglow/sqlite`'s `./storage` used to, but both runtimes now share +the `node:sqlite` driver. That set is pinned by `packages/test/src/test/util/BunExportConditions.test.ts`, which fails in both directions — on a -redundant `"bun"` condition added back, and on one of the three being deleted. +redundant `"bun"` condition added back, and on one of the two being deleted. Each build command follows the same template: @@ -260,9 +260,8 @@ tree-shaking impossible for downstream consumers. ### Extended Pattern (util) -`@workglow/util` has additional entry points beyond the standard two, and accounts for two of the -three `--target=bun` builds that a surviving `"bun"` export condition still earns (`bun.ts` and -`worker-bun.ts`; the third is `@workglow/sqlite`'s `storage/bun.ts`). Each sub-path export +`@workglow/util` has additional entry points beyond the standard two, and is the only package that +still earns a `--target=bun` build — twice (`bun.ts` and `worker-bun.ts`). Each sub-path export gets its own build: ``` @@ -323,15 +322,14 @@ Storage backend packages build one set of entry points per sub-path export: ``` src/storage/browser.ts → dist/storage/browser.js (--target=browser) src/storage/node.ts → dist/storage/node.js (--target=node) -src/storage/bun.ts → dist/storage/bun.js (--target=bun) # @workglow/sqlite only src/job-queue/browser.ts → dist/job-queue/browser.js (--target=browser) src/job-queue/node.ts → dist/job-queue/node.js (--target=node) ``` -`@workglow/sqlite`'s `./storage` is the one sub-path with a real Bun build: it selects `bun:sqlite` -where the Node entry selects the Node driver. Its `./job-queue`, and every sub-path of -`@workglow/postgres` / `@workglow/supabase` / `@workglow/aws` / `@workglow/duckdb`, is -runtime-agnostic on the server and serves Bun from the Node build. +No storage backend has a Bun build. Every server sub-path here — including `@workglow/sqlite`'s +`./storage`, which now drives the built-in `node:sqlite` on both runtimes — is runtime-agnostic +and serves Bun from the Node build, as do all sub-paths of `@workglow/postgres` / +`@workglow/supabase` / `@workglow/aws` / `@workglow/duckdb`. Note that the output directories for sub-paths (`dist/storage/`, `dist/job-queue/`) use `--outdir ./dist/storage` etc. to place them in nested directories matching the sub-path export diff --git a/package.json b/package.json index 64ac9f91d..a24605153 100644 --- a/package.json +++ b/package.json @@ -89,7 +89,6 @@ "@duckdb/node-api": "^1.5.5-r.4", "@sqlite.org/sqlite-wasm": "^3.53.0-build1", "@sqliteai/sqlite-vector": "^1.0.0", - "better-sqlite3": "^13.0.3", "@aws-sdk/client-sqs": "^3.1111.0", "@cloudflare/workers-types": "^5.20260815.1", "fake-indexeddb": "^6.2.4", @@ -138,11 +137,11 @@ "vitest": "catalog:" }, "engines": { - "bun": "^1.3.11" + "bun": "^1.4.0-canary.1", + "node": ">=24" }, - "packageManager": "bun@1.3.11", + "packageManager": "bun@1.4.0-canary.1", "trustedDependencies": [ - "better-sqlite3", "node-llama-cpp", "onnxruntime-node", "protobufjs", diff --git a/packages/storage/README.md b/packages/storage/README.md index ae7b142b1..eaee55270 100644 --- a/packages/storage/README.md +++ b/packages/storage/README.md @@ -141,7 +141,7 @@ import { InMemoryKvStorage, SqliteTabularStorage } from "@workglow/storage"; ### SQLite setup (`@workglow/storage/sqlite`) -Before you open SQLite with a **file path** or construct **`new Sqlite.Database(...)`**, call **`await Sqlite.init()`** once per runtime (Node.js, Bun, or browser). Export is available from **`workglow`** and **`@workglow/storage/sqlite`**. The call is idempotent. On the browser it loads the SQLite WASM build; on Node.js it loads `better-sqlite3`; on Bun it resolves `bun:sqlite`. +Before you open SQLite with a **file path** or construct **`new Sqlite.Database(...)`**, call **`await Sqlite.init()`** once per runtime (Node.js, Bun, or browser). Export is available from **`workglow`** and **`@workglow/storage/sqlite`**. The call is idempotent. On the browser it loads the SQLite WASM build; on Node.js and Bun it loads the built-in `node:sqlite` (Node 24+ / Bun 1.4+). ## Storage Types diff --git a/packages/storage/src/tabular/defineConnectionMutex.ts b/packages/storage/src/tabular/defineConnectionMutex.ts index f14c87848..a6ebe7338 100644 --- a/packages/storage/src/tabular/defineConnectionMutex.ts +++ b/packages/storage/src/tabular/defineConnectionMutex.ts @@ -10,7 +10,7 @@ import type { Als } from "./connectionAls.shared"; * Cross-instance connection safety for storage backends that share a single * underlying database handle across multiple in-process * {@link BaseSqlTabularStorage} instances (e.g. two SqliteTabularStorage - * repositories wrapping the same `better-sqlite3` `Database`, or two + * repositories wrapping the same `node:sqlite` `DatabaseSync`, or two * PostgresTabularStorage repositories over one PGlite session). * * A per-instance mutex only serializes calls that reach a single storage diff --git a/packages/test/src/test/storage-tabular/SqliteDriver.contract.test.ts b/packages/test/src/test/storage-tabular/SqliteDriver.contract.test.ts new file mode 100644 index 000000000..6ebdf11ba --- /dev/null +++ b/packages/test/src/test/storage-tabular/SqliteDriver.contract.test.ts @@ -0,0 +1,309 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Sqlite } from "@workglow/sqlite/storage"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +await Sqlite.init(); + +/** Fresh in-memory database with a single-column `t` table. */ +function makeDb(): Sqlite.Database { + const db = new Sqlite.Database(":memory:"); + db.exec("CREATE TABLE t (id INTEGER PRIMARY KEY)"); + return db; +} + +function countRows(db: Sqlite.Database): number { + const row = db.prepare("SELECT COUNT(*) AS n FROM t").get(); + return row?.n ?? 0; +} + +describe("SQLite driver — transaction()", () => { + let db: Sqlite.Database; + + beforeEach(() => { + db = makeDb(); + }); + + afterEach(() => { + db.close(); + }); + + it("rejects an async body instead of committing it", async () => { + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown): void => { + unhandled.push(reason); + }; + process.on("unhandledRejection", onUnhandled); + try { + const tx = db.transaction(((): unknown => { + db.prepare("INSERT INTO t (id) VALUES (1)").run(); + return (async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + throw new Error("late"); + })(); + }) as () => void); + + expect(() => tx()).toThrow(/cannot return a promise/); + // The wrapper cannot commit work an async body has not finished, so the + // INSERT the body already issued must be rolled back. + expect(countRows(db)).toBe(0); + + // Give the swallowed body time to reject and the process time to notice. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(unhandled).toEqual([]); + } finally { + process.off("unhandledRejection", onUnhandled); + } + }); + + it("commits a synchronous body", () => { + db.transaction(() => { + db.prepare("INSERT INTO t (id) VALUES (1)").run(); + })(); + expect(countRows(db)).toBe(1); + }); + + it("rolls back and rethrows the original error when a synchronous body throws", () => { + const boom = new Error("boom"); + expect(() => + db.transaction(() => { + db.prepare("INSERT INTO t (id) VALUES (1)").run(); + throw boom; + })() + ).toThrow(boom); + expect(countRows(db)).toBe(0); + }); + + it("opens a SAVEPOINT inside a transaction started by exec with a trailing semicolon", () => { + db.exec("BEGIN;"); + db.transaction(() => { + db.prepare("INSERT INTO t (id) VALUES (1)").run(); + })(); + expect(countRows(db)).toBe(1); + + // A nested BEGIN would have thrown above; a SAVEPOINT leaves the outer + // transaction open, so rolling it back must take the row with it. + db.exec("ROLLBACK"); + expect(countRows(db)).toBe(0); + }); + + it("opens a SAVEPOINT inside a transaction started by a prepared BEGIN", () => { + db.prepare("BEGIN").run(); + db.transaction(() => { + db.prepare("INSERT INTO t (id) VALUES (1)").run(); + })(); + expect(countRows(db)).toBe(1); + + db.exec("ROLLBACK"); + expect(countRows(db)).toBe(0); + }); + + it("still commits after a COMMIT that failed because nothing was open", () => { + expect(() => db.exec("COMMIT")).toThrow(); + db.transaction(() => { + db.prepare("INSERT INTO t (id) VALUES (1)").run(); + })(); + expect(countRows(db)).toBe(1); + }); + + it("unwinds only the inner transaction when a nested one throws", () => { + db.transaction(() => { + db.prepare("INSERT INTO t (id) VALUES (1)").run(); + expect(() => + db.transaction(() => { + db.prepare("INSERT INTO t (id) VALUES (2)").run(); + throw new Error("inner"); + })() + ).toThrow("inner"); + db.prepare("INSERT INTO t (id) VALUES (3)").run(); + })(); + + const ids = db + .prepare("SELECT id FROM t ORDER BY id") + .all() + .map((row) => row.id); + expect(ids).toEqual([1, 3]); + }); +}); + +describe("SQLite driver — constructor options", () => { + it.each([ + ["readonly", { readonly: true }], + ["fileMustExist", { fileMustExist: true }], + ["verbose", { verbose: () => {} }], + ["nativeBinding", { nativeBinding: "/nope.node" }], + ["nonsense", { nonsense: 1 }], + ])("rejects the %s option", (_name, options) => { + expect(() => new Sqlite.Database(":memory:", options as never)).toThrow(TypeError); + }); + + it("accepts the supported options", () => { + const db = new Sqlite.Database(":memory:", { readOnly: false, timeout: 1234 }); + db.close(); + }); +}); + +describe("SQLite driver — result rows", () => { + let db: Sqlite.Database; + + beforeEach(() => { + db = makeDb(); + }); + + afterEach(() => { + db.close(); + }); + + it("returns rows with the ordinary object prototype", () => { + const row = db.prepare("SELECT 1 AS x").get(); + expect(Object.getPrototypeOf(row)).toBe(Object.prototype); + expect(Object.prototype.hasOwnProperty.call(row, "x")).toBe(true); + // A null-prototype row makes this throw rather than return a boolean. + expect((row as unknown as { hasOwnProperty(k: string): boolean }).hasOwnProperty("x")).toBe( + true + ); + expect(row).toStrictEqual({ x: 1 }); + }); + + it("returns all() rows with the ordinary object prototype", () => { + db.exec("INSERT INTO t (id) VALUES (1), (2)"); + const rows = db.prepare("SELECT id FROM t ORDER BY id").all(); + expect(rows).toStrictEqual([{ id: 1 }, { id: 2 }]); + for (const row of rows) expect(Object.getPrototypeOf(row)).toBe(Object.prototype); + }); + + it("narrows BigInt integers back to numbers only when they round-trip", () => { + db.exec("CREATE TABLE big (n INTEGER)"); + db.prepare("INSERT INTO big (n) VALUES (?)").run(Number.MAX_SAFE_INTEGER); + db.prepare("INSERT INTO big (n) VALUES (?)").run(BigInt(Number.MAX_SAFE_INTEGER) + 10n); + const rows = db + .prepare("SELECT n FROM big ORDER BY n") + .all(); + expect(rows[0]!.n).toBe(Number.MAX_SAFE_INTEGER); + expect(rows[1]!.n).toBe(BigInt(Number.MAX_SAFE_INTEGER) + 10n); + }); + + it("binds undefined as SQL NULL", () => { + db.exec("CREATE TABLE nullable (id INTEGER PRIMARY KEY, v TEXT)"); + db.prepare("INSERT INTO nullable (id, v) VALUES (?, ?)").run(1, undefined); + const row = db.prepare("SELECT v FROM nullable").get(); + expect(row?.v).toBeNull(); + }); +}); + +describe("SQLite driver — error translation", () => { + it("reports a UNIQUE violation with the better-sqlite3 code spelling", () => { + const db = new Sqlite.Database(":memory:"); + try { + db.exec("CREATE TABLE u (id INTEGER PRIMARY KEY, v TEXT UNIQUE)"); + db.prepare("INSERT INTO u (id, v) VALUES (1, 'a')").run(); + let caught: unknown; + try { + db.prepare("INSERT INTO u (id, v) VALUES (2, 'a')").run(); + } catch (err) { + caught = err; + } + const e = caught as { code?: string; nodeCode?: string }; + expect(e.code).toBe("SQLITE_CONSTRAINT_UNIQUE"); + expect(e.nodeCode).toBe("ERR_SQLITE_ERROR"); + } finally { + db.close(); + } + }); +}); + +describe("SQLite driver — foreign keys", () => { + it("leaves foreign keys off by default", () => { + const db = new Sqlite.Database(":memory:"); + try { + const row = db.prepare("PRAGMA foreign_keys").get(); + expect(row?.foreign_keys).toBe(0); + db.exec("CREATE TABLE parent (id INTEGER PRIMARY KEY)"); + db.exec("CREATE TABLE child (id INTEGER PRIMARY KEY, p INTEGER REFERENCES parent(id))"); + // A dangling reference is what an unenforced schema legitimately holds. + db.prepare("INSERT INTO child (id, p) VALUES (1, 99)").run(); + } finally { + db.close(); + } + }); + + it("enforces foreign keys when asked", () => { + const db = new Sqlite.Database(":memory:", { enableForeignKeyConstraints: true }); + try { + const row = db.prepare("PRAGMA foreign_keys").get(); + expect(row?.foreign_keys).toBe(1); + db.exec("CREATE TABLE parent (id INTEGER PRIMARY KEY)"); + db.exec("CREATE TABLE child (id INTEGER PRIMARY KEY, p INTEGER REFERENCES parent(id))"); + let caught: unknown; + try { + db.prepare("INSERT INTO child (id, p) VALUES (1, 99)").run(); + } catch (err) { + caught = err; + } + expect((caught as { code?: string }).code).toBe("SQLITE_CONSTRAINT_FOREIGNKEY"); + } finally { + db.close(); + } + }); +}); + +describe("SQLite driver — file-backed database", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "workglow-sqlite-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("opens in WAL mode with the shared busy timeout", () => { + const db = new Sqlite.Database(join(dir, "test.db")); + try { + const journal = db.prepare("PRAGMA journal_mode").get(); + expect(journal?.journal_mode).toBe("wal"); + const busy = db.prepare("PRAGMA busy_timeout").get(); + expect(busy?.timeout).toBe(5000); + } finally { + db.close(); + } + }); + + it("waits for a lock held by another connection instead of failing immediately", () => { + const file = join(dir, "contended.db"); + const holder = new Sqlite.Database(file); + const waiter = new Sqlite.Database(file, { timeout: 250 }); + try { + holder.exec("CREATE TABLE t (id INTEGER PRIMARY KEY)"); + holder.exec("BEGIN IMMEDIATE"); + holder.prepare("INSERT INTO t (id) VALUES (1)").run(); + + const start = Date.now(); + let caught: unknown; + try { + waiter.prepare("INSERT INTO t (id) VALUES (2)").run(); + } catch (err) { + caught = err; + } + const waited = Date.now() - start; + + expect((caught as { code?: string }).code).toBe("SQLITE_BUSY"); + // Without a busy timeout the write fails in microseconds; the driver's + // default makes it wait out the configured window first. + expect(waited).toBeGreaterThanOrEqual(200); + + holder.exec("ROLLBACK"); + } finally { + waiter.close(); + holder.close(); + } + }); +}); diff --git a/packages/test/src/test/storage-tabular/SqliteTabularStorage.integration.test.ts b/packages/test/src/test/storage-tabular/SqliteTabularStorage.integration.test.ts index 2d660459b..5f7e38eb6 100644 --- a/packages/test/src/test/storage-tabular/SqliteTabularStorage.integration.test.ts +++ b/packages/test/src/test/storage-tabular/SqliteTabularStorage.integration.test.ts @@ -554,3 +554,48 @@ describe("SqliteTabularStorage shared-connection safety", () => { expect(await a.get({ name: "n3", type: "x" })).toBeDefined(); }); }); + +describe("SqliteTabularStorage entity prototypes", () => { + const row = { + id: "p1", + category: "a", + subcategory: "x", + value: 1, + createdAt: "2025-01-01T00:00:00Z", + updatedAt: "2025-01-01T00:00:00Z", + }; + + async function makeStorage(table: string) { + const storage = new SqliteTabularStorage( + ":memory:", + table, + SearchSchema, + SearchPrimaryKeyNames + ); + await storage.setupDatabase(); + return storage; + } + + it("returns get() entities as plain objects", async () => { + const storage = await makeStorage(`proto_get_${uuid4().replace(/-/g, "_")}`); + await storage.put(row); + const got = await storage.get({ id: "p1" }); + // The driver hands back null-prototype rows; anything that reaches a caller + // has to look like the plain objects every other backend returns. + expect(Object.getPrototypeOf(got)).toBe(Object.prototype); + // Every nullable column SearchSchema declares comes back as an explicit + // null, `tag` included -- a column added to the schema has to be listed + // here or toStrictEqual fails on the extra key. + expect(got).toStrictEqual({ ...row, kind: null, tag: null }); + storage.destroy(); + }); + + it("returns updateWhere() entities as plain objects", async () => { + const storage = await makeStorage(`proto_update_${uuid4().replace(/-/g, "_")}`); + await storage.put(row); + const updated = await storage.updateWhere({ id: "p1" }, { value: 42 }); + expect(Object.getPrototypeOf(updated)).toBe(Object.prototype); + expect(updated).toStrictEqual({ ...row, kind: null, tag: null, value: 42 }); + storage.destroy(); + }); +}); diff --git a/packages/test/src/test/task-graph-output-cache/StreamingSqliteTaskOutputRepository.integration.test.ts b/packages/test/src/test/task-graph-output-cache/StreamingSqliteTaskOutputRepository.integration.test.ts index cd9331c61..a897b6083 100644 --- a/packages/test/src/test/task-graph-output-cache/StreamingSqliteTaskOutputRepository.integration.test.ts +++ b/packages/test/src/test/task-graph-output-cache/StreamingSqliteTaskOutputRepository.integration.test.ts @@ -8,7 +8,7 @@ * `StreamingSqliteTaskOutputRepository` is a durable, embedded streaming cache * backing: JSON rows via `SqliteTabularStorage`, port payloads as ordered BLOB * chunk rows via `TabularBlobChunkStore`. Exercised against an in-memory SQLite - * database (better-sqlite3). + * database (node:sqlite). */ import { Sqlite } from "@workglow/sqlite/storage"; diff --git a/packages/test/src/test/util/BunExportConditions.test.ts b/packages/test/src/test/util/BunExportConditions.test.ts index 8d487206e..4df957920 100644 --- a/packages/test/src/test/util/BunExportConditions.test.ts +++ b/packages/test/src/test/util/BunExportConditions.test.ts @@ -18,15 +18,18 @@ const workspaceRoots = ["packages", "providers"] as const; * * - `@workglow/util .` — `Worker.bun` vs `Worker.node` * - `@workglow/util ./worker` — `dist/worker-bun.js` vs `dist/worker-node.js` - * - `@workglow/sqlite ./storage` — `bun:sqlite` vs the better-sqlite3 driver + * + * `@workglow/sqlite ./storage` was a third until both runtimes moved onto the + * shared `node:sqlite` driver: with no `bun:sqlite` build left to select, the + * condition named the same bundle the default `"import"` already resolves. * * Everywhere else Bun resolves the default `"import"` and loads the node build, * which is byte-identical to what a duplicated `src/bun.ts` would have produced. * * This fixture is deliberately exact, so it fails in both directions: on a - * redundant `"bun"` condition added back, and on one of the three being deleted - * (deleting `@workglow/util`'s `"./worker"` would silently swap every Bun - * consumer onto `Worker.node` and `node:worker_threads`). + * redundant `"bun"` condition added back, and on either of the two being + * deleted (deleting `@workglow/util`'s `"./worker"` would silently swap every + * Bun consumer onto `Worker.node` and `node:worker_threads`). * * Changing it means changing prose too — the same list is stated in * `.claude/CLAUDE.md` ("No `bun` entry unless it differs") and twice in @@ -34,11 +37,7 @@ const workspaceRoots = ["packages", "providers"] as const; * and the "Extended Pattern (util)" build table). `docs/technical/18-multi-runtime-abstraction.md` * also describes `@workglow/util`'s `./worker` condition as the exception. */ -const EXPECTED_BUN_CONDITIONS = [ - "@workglow/sqlite ./storage", - "@workglow/util .", - "@workglow/util ./worker", -] as const; +const EXPECTED_BUN_CONDITIONS = ["@workglow/util .", "@workglow/util ./worker"] as const; interface ExportsNode { readonly [condition: string]: string | ExportsNode | undefined; @@ -104,7 +103,14 @@ describe("bun export conditions", () => { expect(utilExports["."].import).toBe("./dist/node.js"); expect(utilExports["./worker"].bun.import).toBe("./dist/worker-bun.js"); expect(utilExports["./worker"].import).toBe("./dist/worker-node.js"); - expect(sqliteExports["./storage"].bun.import).toBe("./dist/storage/bun.js"); + + // The inverse, kept rather than deleted: `./storage` must carry NO `bun` + // condition and still route to the node build, which is what makes Bun and + // Node share the one `node:sqlite` driver. Dropping the assertion with the + // condition would leave the migration's whole point unpinned — a + // reintroduced `bun:sqlite` branch would then only trip the exact-set test + // above, with nothing saying which driver Bun actually loads. + expect(sqliteExports["./storage"].bun).toBeUndefined(); expect(sqliteExports["./storage"].import).toBe("./dist/storage/node.js"); }); }); diff --git a/packages/test/src/test/vector/SqliteAiVectorStorage.integration.test.ts b/packages/test/src/test/vector/SqliteAiVectorStorage.integration.test.ts index 97877d61f..43b44c2d3 100644 --- a/packages/test/src/test/vector/SqliteAiVectorStorage.integration.test.ts +++ b/packages/test/src/test/vector/SqliteAiVectorStorage.integration.test.ts @@ -14,17 +14,24 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; const _require = createRequire(import.meta.url); let sqliteVectorAvailable = false; +let extensionPath: string | undefined; try { // Use CJS require so the platform-specific sub-package resolves correctly in ESM contexts const mod = _require("@sqliteai/sqlite-vector"); + extensionPath = mod.getExtensionPath() as string; +} catch { + // No prebuilt sqlite-vector binary for this platform — the suite skips. +} +if (extensionPath !== undefined) { + // Resolving the package but failing to load it means the driver's extension + // support is broken, not that the platform lacks a binary. Skipping there + // would hide the regression behind a green suite, so let it throw. await Sqlite.init(); const db = new Sqlite.Database(":memory:"); - db.loadExtension(mod.getExtensionPath()); + db.loadExtension(extensionPath); db.exec("SELECT vector_version()"); db.close(); sqliteVectorAvailable = true; -} catch { - // sqlite-vector extension not available } const VectorSchema = { diff --git a/packages/workglow/README.md b/packages/workglow/README.md index a3ebf005a..98f992caf 100644 --- a/packages/workglow/README.md +++ b/packages/workglow/README.md @@ -88,7 +88,7 @@ bun add @mediapipe/tasks-text @mediapipe/tasks-vision @mediapipe/tasks-audio @me # Storage backends bun add @sqlite.org/sqlite-wasm # Browser SQLite -bun add better-sqlite3 # Node.js/Bun SQLite + # Node.js and Bun use the built-in node:sqlite bun add pg # PostgreSQL bun add @supabase/supabase-js # Supabase ``` diff --git a/providers/sqlite/package.json b/providers/sqlite/package.json index 2e929636a..98aab4d84 100644 --- a/providers/sqlite/package.json +++ b/providers/sqlite/package.json @@ -15,19 +15,17 @@ "homepage": "https://workglow.dev", "scripts": { "watch": "concurrently -c 'auto' 'bun:watch-*'", - "watch-js": "concurrently -c 'auto' -n 'storage-br,storage-n,storage-bu,jq-br,jq-n' 'bun run watch-storage-browser' 'bun run watch-storage-node' 'bun run watch-storage-bun' 'bun run watch-jq-browser' 'bun run watch-jq-node'", + "watch-js": "concurrently -c 'auto' -n 'storage-br,storage-n,jq-br,jq-n' 'bun run watch-storage-browser' 'bun run watch-storage-node' 'bun run watch-jq-browser' 'bun run watch-jq-node'", "watch-storage-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --packages=external --outdir ./dist/storage ./src/storage/browser.ts", "watch-storage-node": "bun build --watch --no-clear-screen --target=node --sourcemap=external --packages=external --outdir ./dist/storage ./src/storage/node.ts", - "watch-storage-bun": "bun build --watch --no-clear-screen --target=bun --sourcemap=external --packages=external --outdir ./dist/storage ./src/storage/bun.ts", "watch-jq-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --packages=external --outdir ./dist/job-queue ./src/job-queue/browser.ts", "watch-jq-node": "bun build --watch --no-clear-screen --target=node --sourcemap=external --packages=external --outdir ./dist/job-queue ./src/job-queue/node.ts", "watch-types": "tsc --watch --preserveWatchOutput", "build-package": "bun run build-js && bun run build-types", - "build-js": "concurrently -m 12 --timings -c 'auto' -n 'storage-br,storage-n,storage-bu,jq-br,jq-n' 'bun run build-storage-browser' 'bun run build-storage-node' 'bun run build-storage-bun' 'bun run build-jq-browser' 'bun run build-jq-node'", + "build-js": "concurrently -m 12 --timings -c 'auto' -n 'storage-br,storage-n,jq-br,jq-n' 'bun run build-storage-browser' 'bun run build-storage-node' 'bun run build-jq-browser' 'bun run build-jq-node'", "build-clean": "rm -fr dist/* tsconfig.tsbuildinfo", "build-storage-browser": "bun build --target=browser --sourcemap=external --packages=external --outdir ./dist/storage ./src/storage/browser.ts", "build-storage-node": "bun build --target=node --sourcemap=external --packages=external --outdir ./dist/storage ./src/storage/node.ts", - "build-storage-bun": "bun build --target=bun --sourcemap=external --packages=external --outdir ./dist/storage ./src/storage/bun.ts", "build-jq-browser": "bun build --target=browser --sourcemap=external --packages=external --outdir ./dist/job-queue ./src/job-queue/browser.ts", "build-jq-node": "bun build --target=node --sourcemap=external --packages=external --outdir ./dist/job-queue ./src/job-queue/node.ts", "build-types": "rm -f tsconfig.tsbuildinfo && tsgo", @@ -39,10 +37,6 @@ "types": "./dist/storage/browser.d.ts", "import": "./dist/storage/browser.js" }, - "bun": { - "types": "./dist/storage/bun.d.ts", - "import": "./dist/storage/bun.js" - }, "types": "./dist/storage/node.d.ts", "import": "./dist/storage/node.js" }, @@ -56,7 +50,6 @@ } }, "peerDependencies": { - "better-sqlite3": "catalog:", "@sqlite.org/sqlite-wasm": "catalog:", "@sqliteai/sqlite-vector": "catalog:", "@workglow/job-queue": "workspace:*", @@ -64,9 +57,6 @@ "@workglow/util": "workspace:*" }, "peerDependenciesMeta": { - "better-sqlite3": { - "optional": true - }, "@sqlite.org/sqlite-wasm": { "optional": true }, @@ -86,11 +76,9 @@ "devDependencies": { "@sqlite.org/sqlite-wasm": "catalog:", "@sqliteai/sqlite-vector": "catalog:", - "@types/better-sqlite3": "^9.6.0", "@workglow/job-queue": "workspace:*", "@workglow/storage": "workspace:*", - "@workglow/util": "workspace:*", - "better-sqlite3": "catalog:" + "@workglow/util": "workspace:*" }, "files": [ "dist", diff --git a/providers/sqlite/src/migrations/SqliteMigrationRunner.ts b/providers/sqlite/src/migrations/SqliteMigrationRunner.ts index ba115e1f9..bf0b0083b 100644 --- a/providers/sqlite/src/migrations/SqliteMigrationRunner.ts +++ b/providers/sqlite/src/migrations/SqliteMigrationRunner.ts @@ -36,7 +36,7 @@ const isSqliteConstraintError = (err: unknown): boolean => { * Each migration is wrapped in an explicit `BEGIN`/`COMMIT`/`ROLLBACK` so the * bookkeeping INSERT and the migration's DDL commit together — and a failure * in `up()` rolls back any partial schema it created. We do not use - * better-sqlite3's `transaction()` helper because it cannot span the `await` + * the driver's `transaction()` helper because it cannot span the `await` * boundary that `up()` may introduce; manual BEGIN/COMMIT works because the * surrounding `runLocks` mutex prevents any other migration call from * touching the connection mid-transaction. diff --git a/providers/sqlite/src/storage/SqliteAiVectorStorage.ts b/providers/sqlite/src/storage/SqliteAiVectorStorage.ts index 290fcec48..78701ab92 100644 --- a/providers/sqlite/src/storage/SqliteAiVectorStorage.ts +++ b/providers/sqlite/src/storage/SqliteAiVectorStorage.ts @@ -466,7 +466,7 @@ export class SqliteAiVectorStorage< * Internal `putBulk` that the proxy dispatches to from `tx.putBulk(...)`. * When already inside an outer `withTransaction` BEGIN we cannot open a * second `db.transaction(...)` here — that would be a nested transaction - * (better-sqlite3 turns it into a SAVEPOINT, but we still want all + * (the driver turns it into a SAVEPOINT, but we still want all * row-level rollback to fall under the outer BEGIN). In that case we * iterate the rows directly; otherwise we wrap them in a SQLite * transaction to collapse N fsyncs into one COMMIT. @@ -503,8 +503,8 @@ export class SqliteAiVectorStorage< private async runVectorPutBulkOnHandle(entities: any[]): Promise { const updatedEntities: Entity[] = []; - // better-sqlite3 / bun:sqlite expose `inTransaction` as a runtime getter - // on the underlying handle; the canonical API doesn't surface it. When it's + // Some drivers expose `inTransaction` as a runtime getter on the underlying + // handle; the canonical API doesn't surface it. When it's // present and true (e.g. an outer `withTransaction` BEGIN is open and the // proxy routed `tx.putBulk` here), iterate rows directly so we don't open a // nested SQLite transaction. Browser-WASM has no such flag and never wraps diff --git a/providers/sqlite/src/storage/SqliteTabularStorage.ts b/providers/sqlite/src/storage/SqliteTabularStorage.ts index 81fc3008f..a96a40030 100644 --- a/providers/sqlite/src/storage/SqliteTabularStorage.ts +++ b/providers/sqlite/src/storage/SqliteTabularStorage.ts @@ -488,7 +488,7 @@ export class SqliteTabularStorage< /** * Synchronously inserts a single entity using prepared `INSERT OR REPLACE … * RETURNING *`. Extracted so {@link putBulk} can call it inside a single - * `db.transaction(...)` — better-sqlite3 transactions require a sync body, + * `db.transaction(...)` — driver transactions require a sync body, * and every step here (key generation, statement prep, bind, RETURNING) is * synchronous in our SQLite drivers. * @@ -790,8 +790,8 @@ export class SqliteTabularStorage< // every chunk in one atomic unit. All statement execution is synchronous // sqlite work, so no other writer interleaves between BEGIN and COMMIT. // Prefer our own `inTransaction` flag: the driver-wrapped `Sqlite.Database` - // does not surface better-sqlite3's native `inTransaction` getter, so - // reading through the wrapper misses the nested-tx signal. + // does not surface a native `inTransaction` getter, so reading through the + // wrapper misses the nested-tx signal. const nativeFlag = (this.db as unknown as { inTransaction?: boolean }).inTransaction === true; const alreadyInTx = this.inTransaction || nativeFlag; if (!alreadyInTx) this.db.exec("BEGIN"); @@ -898,7 +898,7 @@ export class SqliteTabularStorage< /** * Runs `fn` inside a single SQLite transaction. Uses raw `BEGIN` / * `COMMIT` / `ROLLBACK` rather than {@link Sqlite.Database.transaction} - * because `fn` is async — better-sqlite3's transaction wrapper requires a + * because `fn` is async — the driver's transaction wrapper requires a * synchronous body. * * Concurrent ops on the same storage instance from *outside* `fn` queue diff --git a/providers/sqlite/src/storage/_sqlite/browser.ts b/providers/sqlite/src/storage/_sqlite/browser.ts index c4e815cc1..9fc5a40f0 100644 --- a/providers/sqlite/src/storage/_sqlite/browser.ts +++ b/providers/sqlite/src/storage/_sqlite/browser.ts @@ -104,7 +104,7 @@ class BrowserStatement< } /** - * better-sqlite3 / {@link Sqlite.Database}–shaped wrapper around sqlite-wasm {@link WasmDatabase}. + * {@link Sqlite.Database}–shaped wrapper around sqlite-wasm {@link WasmDatabase}. */ export class BrowserDatabase implements SqliteApi.Database { private readonly inner: WasmDatabase; @@ -130,7 +130,7 @@ export class BrowserDatabase implements SqliteApi.Database { } /** - * Same contract as better-sqlite3 / Bun: returns a function that runs `fn` inside a single + * Same contract as the Node and Bun drivers: returns a function that runs `fn` inside a single * SQL transaction (BEGIN → COMMIT or ROLLBACK). */ transaction(fn: (...args: T) => void): (...args: T) => void { diff --git a/providers/sqlite/src/storage/_sqlite/bun.ts b/providers/sqlite/src/storage/_sqlite/bun.ts deleted file mode 100644 index bc66c8d8e..000000000 --- a/providers/sqlite/src/storage/_sqlite/bun.ts +++ /dev/null @@ -1,131 +0,0 @@ -/** - * @license - * Copyright 2025 Steven Roussey - * SPDX-License-Identifier: Apache-2.0 - */ - -import type { Database as BunDatabaseCtor, Statement as BunStatementType } from "bun:sqlite"; - -import type { SqliteApi } from "./canonical-api"; -import { SQLITE_BUSY_TIMEOUT_MS } from "./canonical-api"; - -export type { SqliteApi }; - -type BunSqliteModule = typeof import("bun:sqlite"); - -let _bunSqlite: BunSqliteModule | undefined; -let initPromise: Promise | undefined; - -function assertBunLoaded(): BunSqliteModule { - if (!_bunSqlite) { - throw new Error("SQLite is not ready. Await Sqlite.init() before using new Sqlite.Database()."); - } - return _bunSqlite; -} - -/** - * Resolves `bun:sqlite` via dynamic import. Idempotent; concurrent callers share one load. - */ -function initSqlite(): Promise { - return (initPromise ??= (async () => { - if (_bunSqlite) { - return; - } - _bunSqlite = await import("bun:sqlite"); - })()); -} - -function getBunSqlite(): BunSqliteModule { - return assertBunLoaded(); -} - -function toRunResult(changes: number, lastInsertRowid: number | bigint): SqliteApi.RunResult { - return { changes, lastInsertRowid }; -} - -class BunStatementAdapter< - BindParameters extends unknown[] | Record = unknown[], - Result = unknown, -> implements SqliteApi.Statement { - readonly #stmt: BunStatementType; - - constructor(stmt: BunStatementType) { - this.#stmt = stmt; - } - - run(...params: unknown[]): SqliteApi.RunResult { - const meta = this.#stmt.run(...(params as never[])); - return toRunResult(meta.changes, meta.lastInsertRowid); - } - - get(...params: unknown[]): Result | undefined { - const row = this.#stmt.get(...(params as never[])); - return row === null ? undefined : row; - } - - all(...params: unknown[]): Result[] { - return this.#stmt.all(...(params as never[])); - } - - finalize(): void { - this.#stmt.finalize(); - } -} - -/** - * Bun `bun:sqlite` database wrapped to match {@link SqliteApi.Database}: - * `prepare` uses bindings-first generics; `get()` maps `null` → `undefined`. - */ -export class BunSqliteDatabase implements SqliteApi.Database { - readonly #db: InstanceType; - - constructor(filename?: string, options?: number | import("bun:sqlite").DatabaseOptions) { - const { Database } = getBunSqlite(); - this.#db = new Database(filename, options); - // bun:sqlite has no default busy_timeout; set our shared default so a - // contended write waits for the lock instead of failing with SQLITE_BUSY. - this.#db.run(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`); - // WAL gives readers and writers concurrency across the several connections - // that open the same DB file. It needs a real file — skip in-memory dbs, - // where journal_mode stays "memory". - if (filename && filename !== ":memory:") { - this.#db.run("PRAGMA journal_mode = WAL"); - } - } - - exec(sql: string): void { - this.#db.run(sql); - } - - prepare = unknown[], Result = unknown>( - sql: string - ): SqliteApi.Statement { - const stmt = this.#db.prepare(sql); - return new BunStatementAdapter(stmt); - } - - transaction(fn: (...args: T) => void): (...args: T) => void { - const tx = this.#db.transaction(fn); - return (...args: T) => { - tx(...args); - }; - } - - close(): void { - this.#db.close(); - } - - loadExtension(path: string, entryPoint?: string): void { - this.#db.loadExtension(path, entryPoint); - } -} - -export const Sqlite = { - init: initSqlite, - Database: BunSqliteDatabase, -} as const; - -// eslint-disable-next-line @typescript-eslint/no-namespace -export namespace Sqlite { - export type Database = BunSqliteDatabase; -} diff --git a/providers/sqlite/src/storage/_sqlite/canonical-api.ts b/providers/sqlite/src/storage/_sqlite/canonical-api.ts index ca2055cd6..29975a834 100644 --- a/providers/sqlite/src/storage/_sqlite/canonical-api.ts +++ b/providers/sqlite/src/storage/_sqlite/canonical-api.ts @@ -5,13 +5,13 @@ */ /** - * Canonical SQLite surface for `@workglow/sqlite/storage` across Node (better-sqlite3), - * Bun (native, via adapter), and browser (WASM). + * Canonical SQLite surface for `@workglow/sqlite/storage` across Node and Bun + * (both on the built-in `node:sqlite`) and the browser (WASM). * * On every platform, call `await Sqlite.init()` once before `new Sqlite.Database(...)`. * * **Generic order:** `prepare(sql)` — bindings first, - * row/result second (better-sqlite3 order), not `bun:sqlite`’s reversed order. + * row/result second. */ /** @@ -36,6 +36,12 @@ export namespace SqliteApi { run(...params: unknown[]): RunResult; get(...params: unknown[]): Result | undefined; all(...params: unknown[]): Result[]; + /** + * Releases the statement where the driver supports it. On `node:sqlite` + * this is a no-op today: there is no explicit finalizer and `StatementSync` + * exposes no `Symbol.dispose`, so statements are released only on GC or + * when the database closes. Only the browser (WASM) driver frees eagerly. + */ finalize(): void; } diff --git a/providers/sqlite/src/storage/_sqlite/node.ts b/providers/sqlite/src/storage/_sqlite/node.ts index 7a45a344f..5a3c46bf5 100644 --- a/providers/sqlite/src/storage/_sqlite/node.ts +++ b/providers/sqlite/src/storage/_sqlite/node.ts @@ -4,100 +4,457 @@ * SPDX-License-Identifier: Apache-2.0 */ -import type BetterSqlite3 from "better-sqlite3"; - import type { SqliteApi } from "./canonical-api"; import { SQLITE_BUSY_TIMEOUT_MS } from "./canonical-api"; export type { SqliteApi }; -type BetterDatabase = InstanceType; +type NodeSqliteModule = typeof import("node:sqlite"); +type NodeDatabaseSync = InstanceType; +type NodeStatementSync = ReturnType; + +/** + * Constructor options forwarded to `node:sqlite`'s `DatabaseSync`. + * + * Declared structurally rather than re-exporting the built-in `DatabaseSyncOptions` + * so this module keeps compiling against `@types/node` releases that rename or + * extend it. + */ +export interface NodeSqliteOptions { + readonly open?: boolean; + readonly readOnly?: boolean; + /** + * Enforce `REFERENCES` constraints. Defaults to `false`. + * + * `node:sqlite` defaults this to `true`, but SQLite itself — and both drivers + * this one replaces — leave foreign keys off. Schemas written under that + * default legitimately hold rows whose references no longer resolve, and + * delete orders that were legal when they ran, so enforcement stays opt-in: + * turn it on per database once its data and write order satisfy it. + */ + readonly enableForeignKeyConstraints?: boolean; + readonly enableDoubleQuotedStringLiterals?: boolean; + /** Defaults to `true` so `loadExtension` works, matching better-sqlite3. */ + readonly allowExtension?: boolean; + /** `busy_timeout` in ms. Defaults to {@link SQLITE_BUSY_TIMEOUT_MS}. */ + readonly timeout?: number; +} + +/** Keys {@link NodeSqliteOptions} forwards to `DatabaseSync`. */ +const ALLOWED_OPTIONS: ReadonlySet = new Set([ + "open", + "readOnly", + "enableForeignKeyConstraints", + "enableDoubleQuotedStringLiterals", + "allowExtension", + "timeout", +]); + +/** + * better-sqlite3 option names `node:sqlite` renamed or does not have. + * + * `DatabaseSync` ignores unknown constructor keys, so a leftover + * `readonly: true` would open the database read-**write** and a + * `fileMustExist: true` would create the missing file — both silently. + * better-sqlite3 threw on a misspelled option; the seam keeps that, adding the + * migration guidance. + */ +const LEGACY_OPTIONS: Readonly> = { + readonly: "use `readOnly`", + fileMustExist: "no node:sqlite equivalent; check the file exists before opening", + verbose: "no node:sqlite equivalent", + nativeBinding: "no node:sqlite equivalent", +}; + +function assertKnownOptions(options: NodeSqliteOptions | undefined): void { + if (options === undefined) return; + for (const key of Object.keys(options)) { + const guidance = LEGACY_OPTIONS[key]; + if (guidance !== undefined) { + throw new TypeError(`Unsupported better-sqlite3 option "${key}": ${guidance}.`); + } + if (!ALLOWED_OPTIONS.has(key)) { + throw new TypeError( + `Unknown SQLite option "${key}". Supported options: ${[...ALLOWED_OPTIONS].join(", ")}.` + ); + } + } +} + +/** + * Rejects an async {@link SqliteApi.Database.transaction} body. + * + * The wrapper commits as soon as `fn` returns, so an `async` body would commit + * at its first `await` and a later throw could not roll anything back — + * better-sqlite3 refused such a body outright and callers here still rely on + * that rejection. The returned promise is given a no-op `catch` first: the + * body's own eventual rejection is nobody's to handle once its transaction is + * gone, and left alone it would surface as an unhandled rejection on top of + * this `TypeError`. + */ +function assertSyncTransactionBody(result: unknown): void { + if ( + result != null && + (typeof result === "object" || typeof result === "function") && + typeof (result as { then?: unknown }).then === "function" + ) { + void Promise.resolve(result).catch(() => {}); + throw new TypeError("Transaction function cannot return a promise"); + } +} -let BetterCtor: typeof BetterSqlite3 | undefined; +let sqliteModule: NodeSqliteModule | undefined; let initPromise: Promise | undefined; -function assertLoaded(): typeof BetterSqlite3 { - if (!BetterCtor) { +function assertLoaded(): NodeSqliteModule { + if (!sqliteModule) { throw new Error("SQLite is not ready. Await Sqlite.init() before using new Sqlite.Database()."); } - return BetterCtor; + return sqliteModule; } /** - * Loads better-sqlite3 via dynamic import. Idempotent; concurrent callers share one load. + * Loads `node:sqlite` via dynamic import. Idempotent; concurrent callers share one load. */ function initSqlite(): Promise { return (initPromise ??= (async () => { - if (BetterCtor) { + if (sqliteModule) { return; } try { - const mod = await import("better-sqlite3"); - BetterCtor = - (mod as { default?: typeof BetterSqlite3 }).default ?? - (mod as unknown as typeof BetterSqlite3); + sqliteModule = await import("node:sqlite"); } catch { throw new Error( - "better-sqlite3 is required for @workglow/sqlite/storage on Node.js. Install it with: bun add better-sqlite3" + "The built-in node:sqlite module is required for @workglow/sqlite/storage. " + + "It is stable in Node 24+ (on Node 22, run with --experimental-sqlite) " + + "and available in Bun 1.4+." ); } })()); } /** - * better-sqlite3 database wrapped as {@link SqliteApi.Database} (bindings-first `prepare` - * generics). Construct only after {@link Sqlite.init}. + * SQLite extended result code → better-sqlite3-style `code` string. + * + * `node:sqlite` reports every failure as `code: "ERR_SQLITE_ERROR"` and puts the + * SQLite result code on `errcode`. Callers across this repo (and downstream + * packages) branch on the better-sqlite3 spelling — `SQLITE_CONSTRAINT_UNIQUE`, + * `code.startsWith("SQLITE_CONSTRAINT")` — so the driver seam restores it. + * Only the codes that are actually branched on are enumerated; anything else + * falls back to the primary code name via {@link PRIMARY_RESULT_CODES}. + */ +const EXTENDED_RESULT_CODES: Readonly> = { + 275: "SQLITE_CONSTRAINT_CHECK", + 531: "SQLITE_CONSTRAINT_COMMITHOOK", + 787: "SQLITE_CONSTRAINT_FOREIGNKEY", + 1043: "SQLITE_CONSTRAINT_FUNCTION", + 1299: "SQLITE_CONSTRAINT_NOTNULL", + 1555: "SQLITE_CONSTRAINT_PRIMARYKEY", + 1811: "SQLITE_CONSTRAINT_TRIGGER", + 2067: "SQLITE_CONSTRAINT_UNIQUE", + 2323: "SQLITE_CONSTRAINT_VTAB", + 2579: "SQLITE_CONSTRAINT_ROWID", + 2835: "SQLITE_CONSTRAINT_PINNED", + 3091: "SQLITE_CONSTRAINT_DATATYPE", + 261: "SQLITE_BUSY_RECOVERY", + 517: "SQLITE_BUSY_SNAPSHOT", + 773: "SQLITE_BUSY_TIMEOUT", + 516: "SQLITE_READONLY_ROLLBACK", +}; + +/** Primary (low byte) result codes, used when the extended code is unmapped. */ +const PRIMARY_RESULT_CODES: Readonly> = { + 1: "SQLITE_ERROR", + 2: "SQLITE_INTERNAL", + 3: "SQLITE_PERM", + 4: "SQLITE_ABORT", + 5: "SQLITE_BUSY", + 6: "SQLITE_LOCKED", + 7: "SQLITE_NOMEM", + 8: "SQLITE_READONLY", + 9: "SQLITE_INTERRUPT", + 10: "SQLITE_IOERR", + 11: "SQLITE_CORRUPT", + 12: "SQLITE_NOTFOUND", + 13: "SQLITE_FULL", + 14: "SQLITE_CANTOPEN", + 15: "SQLITE_PROTOCOL", + 17: "SQLITE_SCHEMA", + 18: "SQLITE_TOOBIG", + 19: "SQLITE_CONSTRAINT", + 20: "SQLITE_MISMATCH", + 21: "SQLITE_MISUSE", + 23: "SQLITE_AUTH", + 25: "SQLITE_RANGE", + 26: "SQLITE_NOTADB", +}; + +function sqliteCodeName(errcode: number): string | undefined { + return EXTENDED_RESULT_CODES[errcode] ?? PRIMARY_RESULT_CODES[errcode & 0xff]; +} + +/** + * Rewrites `code` on a `node:sqlite` error to its better-sqlite3 spelling, + * preserving the original error (class, message, stack) and stashing Node's own + * `ERR_SQLITE_ERROR` on `nodeCode`. Non-SQLite errors pass through untouched. + */ +function translateError(err: unknown): unknown { + if (err === null || typeof err !== "object") return err; + const e = err as { code?: unknown; errcode?: unknown; nodeCode?: unknown }; + if (e.code !== "ERR_SQLITE_ERROR" || typeof e.errcode !== "number") return err; + const name = sqliteCodeName(e.errcode); + if (name === undefined) return err; + e.nodeCode = e.code; + e.code = name; + return err; +} + +function rethrow(err: unknown): never { + throw translateError(err); +} + +const MIN_SAFE = BigInt(Number.MIN_SAFE_INTEGER); +const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER); + +/** Back to a JS number when it round-trips exactly; otherwise keep the BigInt. */ +function narrowBigInt(value: bigint): number | bigint { + return value >= MIN_SAFE && value <= MAX_SAFE ? Number(value) : value; +} + +/** + * Copies a result row onto a plain object, narrowing its BigInt cells. + * + * `node:sqlite` builds rows with `Object.create(null)`, and those rows are + * handed straight back to callers as entities — where a missing prototype makes + * `row.hasOwnProperty(...)` throw and `toStrictEqual` fail against the plain + * objects every other storage backend returns. Copying restores + * `Object.prototype` and lets the storage layer keep mutating its result. Only + * INTEGER columns arrive as BigInt; REAL, TEXT and BLOB pass through. + */ +function narrowRow(row: unknown): unknown { + if (row === null || typeof row !== "object") return row; + const record = row as Record; + const out: Record = {}; + for (const column in record) { + const value = record[column]; + out[column] = typeof value === "bigint" ? narrowBigInt(value) : value; + } + return out; +} + +/** + * Substitutes `null` for `undefined` bindings. + * + * better-sqlite3 bound `undefined` as SQL NULL; `node:sqlite` rejects it with + * "Provided value cannot be bound to SQLite parameter N". Callers rely on the + * old behavior for columns they leave unset, so the seam keeps it. The input + * array is returned untouched unless it actually holds an `undefined`. + */ +function toBindable(params: unknown[]): unknown[] { + for (let i = 0; i < params.length; i++) { + if (params[i] === undefined) { + return params.map((param) => (param === undefined ? null : param)); + } + } + return params; +} + +/** + * `node:sqlite` `StatementSync` wrapped as {@link SqliteApi.Statement}. + * + * Every statement reads integers as BigInt and narrows them back on the way + * out. Left on its default, `node:sqlite` throws `ERR_OUT_OF_RANGE` for any + * INTEGER past `Number.MAX_SAFE_INTEGER` — and the bulk-put path reads its rows + * back through `INSERT … RETURNING *`, so recovering after the throw would mean + * re-executing the write. Reading wide and narrowing costs one pass over the + * columns already walked downstream, and keeps large keys exact instead of + * silently rounding them the way better-sqlite3's default did. + */ +class NodeSqliteStatement< + BindParameters extends unknown[] | Record = unknown[], + Result = unknown, +> implements SqliteApi.Statement { + readonly #stmt: NodeStatementSync; + + constructor(stmt: NodeStatementSync) { + this.#stmt = stmt; + stmt.setReadBigInts(true); + } + + run(...params: unknown[]): SqliteApi.RunResult { + try { + const { changes, lastInsertRowid } = this.#stmt.run(...(toBindable(params) as never[])); + return { changes: Number(changes), lastInsertRowid: narrowBigInt(BigInt(lastInsertRowid)) }; + } catch (err) { + rethrow(err); + } + } + + get(...params: unknown[]): Result | undefined { + try { + return narrowRow(this.#stmt.get(...(toBindable(params) as never[]))) as Result | undefined; + } catch (err) { + rethrow(err); + } + } + + all(...params: unknown[]): Result[] { + try { + const rows = this.#stmt.all(...(toBindable(params) as never[])); + return rows.map(narrowRow) as Result[]; + } catch (err) { + rethrow(err); + } + } + + /** + * No-op on every `node:sqlite` shipping today: there is no explicit statement + * finalizer, and `StatementSync` carries no `Symbol.dispose` either, so a + * `StatementSync` is released only on GC or when its database closes. The + * optional call stays so a runtime that adds disposal starts using it. + */ + finalize(): void { + (this.#stmt as unknown as Partial)[Symbol.dispose]?.(); + } +} + +/** + * `node:sqlite` database wrapped as {@link SqliteApi.Database} (bindings-first + * `prepare` generics). Construct only after {@link Sqlite.init}. + * + * Shared by the Node and Bun entry points — both runtimes expose `node:sqlite` + * with identical semantics. */ export class NodeSqliteDatabase implements SqliteApi.Database { - readonly #inner: BetterDatabase; + readonly #inner: NodeDatabaseSync; + /** + * Monotonic savepoint counter. Never reused, so a name can never collide with + * a savepoint left open by a failed RELEASE. + */ + #savepointSeq = 0; - constructor(filename?: string, options?: BetterSqlite3.Options) { - const Ctor = assertLoaded(); + constructor(filename?: string, options?: NodeSqliteOptions) { + assertKnownOptions(options); + const { DatabaseSync } = assertLoaded(); const resolved = filename ?? ":memory:"; - this.#inner = new Ctor(resolved, options); - // better-sqlite3 maps `options.timeout` to busy_timeout; when the caller - // didn't specify one, set our shared default so it matches the Bun driver. - if (options?.timeout === undefined) { - this.#inner.pragma(`busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`); + try { + this.#inner = new DatabaseSync(resolved, { + // better-sqlite3 permits loadExtension unconditionally; node:sqlite + // gates it behind this flag, so default it on to keep parity (the + // sqlite-vector extension in SqliteAiVectorStorage depends on it). + allowExtension: true, + ...options, + // node:sqlite turns foreign keys on; SQLite's own default (and every + // driver these databases were written under) leaves them off. + enableForeignKeyConstraints: options?.enableForeignKeyConstraints ?? false, + // node:sqlite defaults busy_timeout to 0, so a contended write fails + // immediately with SQLITE_BUSY instead of waiting for the lock. + timeout: options?.timeout ?? SQLITE_BUSY_TIMEOUT_MS, + }); + } catch (err) { + rethrow(err); + } + if (typeof this.#inner.isTransaction !== "boolean") { + throw new Error( + "node:sqlite is too old: DatabaseSync.isTransaction is required by @workglow/sqlite/storage." + ); } // WAL gives readers and writers concurrency across the several connections // that open the same DB file. It needs a real file — skip in-memory dbs, // where journal_mode stays "memory". - if (resolved !== ":memory:") { - this.#inner.pragma("journal_mode = WAL"); + if (resolved !== ":memory:" && options?.open !== false) { + this.#exec("PRAGMA journal_mode = WAL"); + } + } + + /** Runs `sql`, translating the SQLite error code on failure. */ + #exec(sql: string): void { + try { + this.#inner.exec(sql); + } catch (err) { + rethrow(err); } } exec(sql: string): void { - this.#inner.exec(sql); + this.#exec(sql); } prepare = unknown[], Result = unknown>( sql: string ): SqliteApi.Statement { - return this.#inner.prepare(sql) as unknown as SqliteApi.Statement; + try { + return new NodeSqliteStatement(this.#inner.prepare(sql)); + } catch (err) { + rethrow(err); + } } + /** + * Same contract as better-sqlite3's `transaction()`: returns a function that + * runs `fn` inside a single SQL transaction. A call nested inside an open + * transaction uses a SAVEPOINT, matching better-sqlite3. + * + * Nesting is decided by SQLite's own `isTransaction`, so a transaction any + * caller opened — through this method, through `exec("BEGIN;")`, or through a + * prepared `BEGIN` — is seen, and a failed COMMIT cannot leave the driver + * believing in a transaction the database has already ended. + */ transaction(fn: (...args: T) => void): (...args: T) => void { - const tx = this.#inner.transaction(fn); return (...args: T) => { - tx(...args); + if (this.#inner.isTransaction) { + this.#runInSavepoint(fn, args); + return; + } + this.#exec("BEGIN"); + try { + assertSyncTransactionBody(fn(...args)); + this.#exec("COMMIT"); + } catch (err) { + try { + if (this.#inner.isTransaction) this.#exec("ROLLBACK"); + } catch { + // prefer the original error if rollback fails + } + rethrow(err); + } }; } + #runInSavepoint(fn: (...args: T) => void, args: T): void { + const name = `_workglow_sp_${this.#savepointSeq++}`; + this.#exec(`SAVEPOINT ${name}`); + try { + assertSyncTransactionBody(fn(...args)); + this.#exec(`RELEASE ${name}`); + } catch (err) { + try { + this.#exec(`ROLLBACK TO ${name}`); + this.#exec(`RELEASE ${name}`); + } catch { + // prefer the original error if the savepoint unwind fails + } + rethrow(err); + } + } + close(): void { this.#inner.close(); } loadExtension(path: string, entryPoint?: string): void { - if (entryPoint === undefined) { - this.#inner.loadExtension(path); - } else { - (this.#inner as unknown as { loadExtension(p: string, e?: string): void }).loadExtension( - path, - entryPoint + if (entryPoint !== undefined) { + // node:sqlite's loadExtension takes no entry-point argument and would + // silently load the default entry point instead of the requested one. + throw new Error( + "node:sqlite does not support a loadExtension entryPoint; omit it to use the default." ); } + try { + this.#inner.loadExtension(path); + } catch (err) { + rethrow(err); + } } } diff --git a/providers/sqlite/src/storage/bun.ts b/providers/sqlite/src/storage/bun.ts deleted file mode 100644 index 215d15a0d..000000000 --- a/providers/sqlite/src/storage/bun.ts +++ /dev/null @@ -1,10 +0,0 @@ -/** - * @license - * Copyright 2025 Steven Roussey - * SPDX-License-Identifier: Apache-2.0 - */ - -// organize-imports-ignore - -export * from "./_sqlite/bun"; -export * from "./common";