From d816783a67f1299241d04e8b637d90c8f425e243 Mon Sep 17 00:00:00 2001 From: tedspare Date: Thu, 10 Oct 2024 10:00:46 -0400 Subject: [PATCH] Move DB to top-level --- CHANGELOG.md | 1 + package.json | 2 +- src/evals/index.ts | 13 +++++++++++-- src/evals/multi-turn/index.ts | 16 +++++----------- src/evals/one-shot/index.ts | 9 +++++---- src/types/index.ts | 4 ++++ 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 890b722..35b083d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ +- [2024-10-10] [Move DB to top-level](https://github.com/RubricLab/memory/commit/67913408246413448008f902963eb5a288b963d8) - [2024-10-10] [Decouple extraction and save](https://github.com/RubricLab/memory/commit/d5fd7c08eff9458b9775ef439b480cd38922759d) - [2024-10-09] [Add multi-turn examples](https://github.com/RubricLab/memory/commit/a0e32260e510a3ef1312030fcb5e2685ac2bb300) - [2024-10-09] [Scaffold multi-turn evals](https://github.com/RubricLab/memory/commit/ecb5531acef6a924b81684660150eeb71d93e704) diff --git a/package.json b/package.json index e30a0a3..14e6421 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@rubriclab/memory", "module": "src/index.ts", - "version": "0.0.12", + "version": "0.0.13", "private": false, "type": "module", "devDependencies": { diff --git a/src/evals/index.ts b/src/evals/index.ts index 19f7b9a..fd770f6 100644 --- a/src/evals/index.ts +++ b/src/evals/index.ts @@ -1,3 +1,4 @@ +import Database from 'bun:sqlite' import { parseArgs } from 'node:util' import { runMultiTurnExamples } from '@/evals/multi-turn' import { runOneShotExamples } from '@/evals/one-shot' @@ -37,9 +38,17 @@ if (import.meta.path === Bun.main) { const model = fast ? 'gpt-4o-mini' : 'gpt-4o-2024-08-06' + const db = new Database(':memory:', { create: true, strict: true }) + + await db + .prepare( + 'create table if not exists facts (subject text, relation text, object text, primary key (subject, object))' + ) + .get() + if (dataset === '1') { - await runOneShotExamples({ model }) + await runOneShotExamples({ db, model }) } else if (dataset === '2') { - await runMultiTurnExamples({ model }) + await runMultiTurnExamples({ db, model }) } } diff --git a/src/evals/multi-turn/index.ts b/src/evals/multi-turn/index.ts index 0f7d308..e13d561 100644 --- a/src/evals/multi-turn/index.ts +++ b/src/evals/multi-turn/index.ts @@ -1,20 +1,14 @@ -import { Database } from 'bun:sqlite' import { Memory } from '@/index' -import type { Fact } from '@/types' +import type { Database, Fact } from '@/types' import { format } from '@/utils/string' import type { openai } from '@ai-sdk/openai' import chalk from 'chalk' import { EXAMPLES } from './examples' -const db = new Database(':memory:', { create: true, strict: true }) - -await db - .prepare( - 'create table if not exists facts (subject text, relation text, object text, primary key (subject, object))' - ) - .get() - -export const runMultiTurnExamples = async ({ model }: { model: Parameters[0] }) => { +export const runMultiTurnExamples = async ({ + db, + model +}: { model: Parameters[0]; db: Database }) => { const memory = new Memory({ model, db }) let totalFacts = 0 diff --git a/src/evals/one-shot/index.ts b/src/evals/one-shot/index.ts index 6e52447..b152967 100644 --- a/src/evals/one-shot/index.ts +++ b/src/evals/one-shot/index.ts @@ -1,13 +1,14 @@ -import { Database } from 'bun:sqlite' import { Memory } from '@/index' +import type { Database } from '@/types' import { format } from '@/utils/string' import type { openai } from '@ai-sdk/openai' import chalk from 'chalk' import { EXAMPLES } from './examples' -const db = new Database(':memory:', { create: true, strict: true }) - -export const runOneShotExamples = async ({ model }: { model: Parameters[0] }) => { +export const runOneShotExamples = async ({ + db, + model +}: { model: Parameters[0]; db: Database }) => { const memory = new Memory({ model, db }) let totalFacts = 0 diff --git a/src/types/index.ts b/src/types/index.ts index cae7666..6a01e8d 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,6 +1,10 @@ +import type { Database as BunDatabase } from 'bun:sqlite' + export type Fact = { subject: string relation: string object: string data?: Record } + +export type Database = BunDatabase