Fix precision calc

CHANGELOG.md

@@ -1,3 +1,4 @@
+- [2024-10-09] [Fix precision calc](https://github.com/RubricLab/memory/commit/52fc41e151c47e276c37a24b3489ba414d032a0b)
 - [2024-10-09] [Generalize eval arch](https://github.com/RubricLab/memory/commit/bf80487850e840525a1521925a439d7d9fc8d638)
 - [2024-10-09] [Add help cmd](https://github.com/RubricLab/memory/commit/9f35d0016dcd5d0f909cb77c2ea33ef70da60fb1)
 - [2024-10-09] [Scaffold evals](https://github.com/RubricLab/memory/commit/3801514a795881c74ea225d02eeae001a07ee57a)

evals/one-shot/index.ts

@@ -11,8 +11,6 @@ export const runOneShotExamples = async ({ fast }: { fast?: boolean }) => {
 	let totalAttempts = 0
 
 	for await (const eg of EXAMPLES) {
-		let correctFacts = 0
-
 		totalFacts += eg.facts.length
 
 		console.log(chalk.yellow(`\n\n"${eg.content}"`))
@@ -56,6 +54,8 @@ export const runOneShotExamples = async ({ fast }: { fast?: boolean }) => {
 		const omitted: number[] = []
 
 		for (const [i, fact] of eg.facts.entries()) {
+			let correctFacts = 0
+
 			console.log(
 				`\n🎯 ${i + 1} of ${eg.facts.length}: ${chalk.magenta(fact.subject)} ${chalk.yellow(fact.relation)} ${chalk.blue(fact.object)}`
 			)
@@ -67,23 +67,25 @@ export const runOneShotExamples = async ({ fast }: { fast?: boolean }) => {
 				const correctRelation = fact.relation === relation
 				const correctObject = fact.object === object
 
-				if (omitted.includes(j)) continue
 				console.log(
 					`🤖 ${j + 1} of ${attempts.length}: ${chalk.magenta(format(subject, correctSubject))} ${chalk.yellow(
 						format(relation, correctRelation)
 					)} ${chalk.blue(format(object, correctObject))}`
 				)
 
+				if (omitted.includes(j)) continue
+
 				correctFacts += Number(correctSubject && correctRelation && correctObject)
 
 				if (correctFacts) {
 					omitted.push(j)
 					break
 				}
 			}
+
+			totalRecall += correctFacts
 		}
 
-		totalRecall += correctFacts
 		totalAttempts += attempts.length
 	}
 

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@rubriclab/memory",
   "module": "index.ts",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "private": false,
   "type": "module",
   "devDependencies": {