From 050d270dd8e949e75db7cd3090ca355d8e836b06 Mon Sep 17 00:00:00 2001
From: tedspare <ted.spare@gmail.com>
Date: Wed, 9 Oct 2024 12:52:38 -0400
Subject: [PATCH] Fix precision calc

---
 CHANGELOG.md            |  1 +
 evals/one-shot/index.ts | 10 ++++++----
 package.json            |  2 +-
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 34e942a..e4447a5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,4 @@
+- [2024-10-09] [Fix precision calc](https://github.com/RubricLab/memory/commit/52fc41e151c47e276c37a24b3489ba414d032a0b)
 - [2024-10-09] [Generalize eval arch](https://github.com/RubricLab/memory/commit/bf80487850e840525a1521925a439d7d9fc8d638)
 - [2024-10-09] [Add help cmd](https://github.com/RubricLab/memory/commit/9f35d0016dcd5d0f909cb77c2ea33ef70da60fb1)
 - [2024-10-09] [Scaffold evals](https://github.com/RubricLab/memory/commit/3801514a795881c74ea225d02eeae001a07ee57a)
diff --git a/evals/one-shot/index.ts b/evals/one-shot/index.ts
index 76b67a2..7f7f871 100644
--- a/evals/one-shot/index.ts
+++ b/evals/one-shot/index.ts
@@ -11,8 +11,6 @@ export const runOneShotExamples = async ({ fast }: { fast?: boolean }) => {
 	let totalAttempts = 0
 
 	for await (const eg of EXAMPLES) {
-		let correctFacts = 0
-
 		totalFacts += eg.facts.length
 
 		console.log(chalk.yellow(`\n\n"${eg.content}"`))
@@ -56,6 +54,8 @@ export const runOneShotExamples = async ({ fast }: { fast?: boolean }) => {
 		const omitted: number[] = []
 
 		for (const [i, fact] of eg.facts.entries()) {
+			let correctFacts = 0
+
 			console.log(
 				`\n🎯 ${i + 1} of ${eg.facts.length}: ${chalk.magenta(fact.subject)} ${chalk.yellow(fact.relation)} ${chalk.blue(fact.object)}`
 			)
@@ -67,13 +67,14 @@ export const runOneShotExamples = async ({ fast }: { fast?: boolean }) => {
 				const correctRelation = fact.relation === relation
 				const correctObject = fact.object === object
 
-				if (omitted.includes(j)) continue
 				console.log(
 					`🤖 ${j + 1} of ${attempts.length}: ${chalk.magenta(format(subject, correctSubject))} ${chalk.yellow(
 						format(relation, correctRelation)
 					)} ${chalk.blue(format(object, correctObject))}`
 				)
 
+				if (omitted.includes(j)) continue
+
 				correctFacts += Number(correctSubject && correctRelation && correctObject)
 
 				if (correctFacts) {
@@ -81,9 +82,10 @@ export const runOneShotExamples = async ({ fast }: { fast?: boolean }) => {
 					break
 				}
 			}
+
+			totalRecall += correctFacts
 		}
 
-		totalRecall += correctFacts
 		totalAttempts += attempts.length
 	}
 
diff --git a/package.json b/package.json
index 2701a13..afb01fd 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@rubriclab/memory",
   "module": "index.ts",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "private": false,
   "type": "module",
   "devDependencies": {