diff --git a/cli/Cargo.lock b/cli/Cargo.lock index c20764e5..c8cb90a0 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -3741,6 +3741,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.0", + "tempfile", "tokio", "toml", "tracing", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index b5afec05..bd4292bb 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -65,6 +65,7 @@ sha2 = "0.11" [dev-dependencies] quint-connect = "0.1.2" +tempfile = "3" [lints] workspace = true diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index 14f9de54..11d14eac 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -486,6 +486,37 @@ fn resolve_query_retry_policy() -> RetryPolicy { QUERY_RETRY_POLICY } +#[cfg(test)] +thread_local! { + static READ_STATEMENTS_ISSUED: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Record that one [`TursoDb`] read statement was issued on this thread. +#[cfg(test)] +fn note_read_statement_issued() { + READ_STATEMENTS_ISSUED.with(|count| count.set(count.get() + 1)); +} + +/// Run `body`, returning its result together with the number of [`TursoDb`] +/// read statements ([`TursoDb::query`], [`TursoDb::query_values`], +/// [`TursoDb::query_map`]) it issued on the current thread. +/// +/// Each read method bumps the counter once in its synchronous prelude, before +/// the retry wrapper, so a transient retry never inflates the count and the +/// number reflects *logical* read statements, not connection round-trips. +/// Lets a deterministic single-threaded test assert that an operation which +/// must observe one coherent database snapshot — for example +/// `MutationTraceStore::load_all_tree_roots`, a single `UNION` statement — +/// issues exactly one, and fail if it is ever reimplemented as several +/// independent `SELECT`s unioned in Rust. Not shared across threads. +#[cfg(test)] +pub(crate) fn count_read_statements(body: impl FnOnce() -> T) -> (T, usize) { + READ_STATEMENTS_ISSUED.with(|count| count.set(0)); + let result = body(); + let issued = READ_STATEMENTS_ISSUED.with(std::cell::Cell::get); + (result, issued) +} + /// Generic Turso database adapter. /// /// Wraps a Turso connection with a tokio current-thread runtime so callers can @@ -645,6 +676,9 @@ impl TursoDb { })?; let operation_name = format!("query {} database", M::db_name()); + #[cfg(test)] + note_read_statement_issued(); + run_with_retry_sync( resolve_query_retry_policy::(), &operation_name, @@ -673,6 +707,9 @@ impl TursoDb { })?; let operation_name = format!("query and fetch {} database values", M::db_name()); + #[cfg(test)] + note_read_statement_issued(); + run_with_retry_sync( resolve_query_retry_policy::(), &operation_name, @@ -810,6 +847,9 @@ impl TursoDb { })?; let operation_name = format!("query and fetch {} database rows", M::db_name()); + #[cfg(test)] + note_read_statement_issued(); + let rows = run_with_retry_sync( resolve_query_retry_policy::(), &operation_name, diff --git a/cli/src/services/mutation_trace/runtime/coordinator.rs b/cli/src/services/mutation_trace/runtime/coordinator.rs index 3cb218ff..4df950cc 100644 --- a/cli/src/services/mutation_trace/runtime/coordinator.rs +++ b/cli/src/services/mutation_trace/runtime/coordinator.rs @@ -179,19 +179,28 @@ pub fn coordinate

( where P: FnOnce() -> anyhow::Result, { - coordinate_inner(repository_root, boundary, open_db, || {}, |_attempt| Ok(())) + coordinate_inner( + repository_root, + boundary, + open_db, + || {}, + |_attempt| {}, + |_attempt| Ok(()), + ) } -fn coordinate_inner( +pub(super) fn coordinate_inner( repository_root: &Path, boundary: &RuntimeBoundary, open_db: P, on_lock_contention: F, + after_load: L, after_recovery: R, ) -> Result where P: FnOnce() -> anyhow::Result, F: FnOnce(), + L: FnMut(u32), R: FnMut(u32) -> Result<()>, { let git_dir = resolve_git_dir(repository_root).map_err(CoordinateError::Other)?; @@ -220,6 +229,7 @@ where boundary, open_db, inherited_external_taint, + after_load, after_recovery, )?; @@ -232,16 +242,18 @@ where } } -fn coordinate_protected( +fn coordinate_protected( repository_root: &Path, git_dir: &Path, boundary: &RuntimeBoundary, open_db: P, inherited_external_taint: bool, + after_load: L, after_recovery: R, ) -> Result where P: FnOnce() -> anyhow::Result, + L: FnMut(u32), R: FnMut(u32) -> Result<()>, { let checkout_id = get_or_create_checkout_id(git_dir).map_err(CoordinateError::Other)?; @@ -257,7 +269,7 @@ where &worktree_id, boundary, inherited_external_taint, - |_attempt| {}, + after_load, after_recovery, ) } @@ -1510,6 +1522,7 @@ mod tests { .send(()) .expect("contention signal channel should still be open"); }, + |_attempt| {}, |_attempt| Ok(()), ); result_tx @@ -1719,6 +1732,7 @@ mod tests { .send(()) .expect("contention signal channel should still be open"); }, + |_attempt| {}, |_attempt| Ok(()), ) }) @@ -2085,6 +2099,7 @@ mod tests { }, ok_db, || {}, + |_attempt| {}, |_attempt| { anyhow::bail!("injected failure after recovery, before the boundary commits") }, diff --git a/cli/src/services/mutation_trace/runtime/git_snapshot.rs b/cli/src/services/mutation_trace/runtime/git_snapshot.rs index 763f5453..dce69861 100644 --- a/cli/src/services/mutation_trace/runtime/git_snapshot.rs +++ b/cli/src/services/mutation_trace/runtime/git_snapshot.rs @@ -1,5 +1,6 @@ +use std::io::Write; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Stdio}; use anyhow::{anyhow, Context, Result}; use uuid::Uuid; @@ -10,6 +11,14 @@ const SCE_RUNTIME_DIR: &str = "sce"; const TMP_INDEX_DIR: &str = "tmp"; const REF_NAMESPACE: &str = "refs/sce/mutation-cursor"; +/// `git for-each-ref` format for pin inventory: four `%00`-separated fields — +/// refname, target object name, target object type, and the symbolic-ref +/// target (empty for a direct ref). NUL-separated so no field can be split or +/// trimmed ambiguously; the trailing symref field is always present (possibly +/// empty), so every well-formed line has exactly four fields. +const FOR_EACH_REF_PIN_FORMAT: &str = + "--format=%(refname)%00%(objectname)%00%(objecttype)%00%(symref)"; + pub struct GitSnapshotService { git_dir: PathBuf, repository_root: PathBuf, @@ -67,6 +76,188 @@ impl GitSnapshotService { ) } + /// Inventory every SCE snapshot pin owned by `worktree_id`. + /// + /// Runs `git for-each-ref` constrained to the single path prefix + /// `refs/sce/mutation-cursor//`, so a ref owned by any other + /// worktree or in an unrelated namespace is never returned. Each line is + /// validated against the shape `pin_tree` produces: a **direct** ref (never + /// a symbolic ref) whose target is a tree object and whose final path + /// component equals the target SHA. A symbolic ref anywhere in the + /// namespace is malformed state — it would let one worktree's pin resolve + /// through another worktree's ref — and is rejected rather than followed. A + /// `git for-each-ref` execution or exit failure is + /// [`PinInventoryError::Git`]; anything malformed inside the namespace is + /// [`PinInventoryError::MalformedRef`], matchable separately. + pub fn list_pins( + &self, + worktree_id: &WorktreeId, + ) -> std::result::Result, PinInventoryError> { + let prefix = pin_ref_prefix(worktree_id); + let raw = self + .run_git(&["for-each-ref", FOR_EACH_REF_PIN_FORMAT, &prefix], None) + .map_err(PinInventoryError::Git)?; + + raw.lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(|line| parse_pin_line(line, &prefix)) + .collect() + } + + /// Delete exactly `pins` in one atomic, no-dereference + /// `git update-ref --no-deref --stdin` transaction, each `delete` + /// conditioned on the tree SHA recorded in the [`PinnedRef`]. + /// + /// Two independent safety properties: + /// + /// - **Atomic** — `git update-ref --stdin` commits every command together + /// at end of input; if any command fails (including a failed old-value + /// check) the whole transaction aborts and no ref is changed. + /// - **No dereference** — `--no-deref` makes every `delete` operate on the + /// exact ref name given, never on a ref reached by resolving a symbolic + /// ref. Combined with a fail-closed re-check (below), a + /// direct-ref → symbolic-ref race between inventory and deletion can + /// never cause this call to touch the symref's target (for example a ref + /// owned by another worktree). + /// + /// Before issuing the transaction, each supplied ref is re-inventoried: it + /// must still exist, still be a direct ref to a tree, and still point at + /// the inventoried SHA. If any has changed — deleted, retargeted, or turned + /// into a symbolic ref — this returns `Err` and deletes nothing, preferring + /// failure over acting on unexpected namespace state. An empty slice is a + /// successful no-op. + pub fn delete_pins(&self, pins: &[PinnedRef]) -> Result<()> { + self.delete_pins_inner(pins, || {}) + } + + /// Body of [`delete_pins`] with a deterministic test seam that fires + /// **after** the fail-closed preflight re-inventory and **before** the + /// `git update-ref --no-deref --stdin` transaction is spawned. Production + /// calls it with a no-op hook; the inline atomicity test uses the hook to + /// mutate a ref *after* it has passed preflight, so the transaction is + /// actually issued and the per-`delete` expected-old-value check — not the + /// preflight — is what aborts the batch. This is the only proof that the + /// Git transaction itself is atomic; the preflight proves a different + /// property (unexpected ref state before the transaction is even attempted). + fn delete_pins_inner(&self, pins: &[PinnedRef], after_preflight: impl FnOnce()) -> Result<()> { + if pins.is_empty() { + return Ok(()); + } + + self.assert_pins_are_unchanged_direct_refs(pins)?; + + after_preflight(); + + let mut stdin_payload = String::new(); + for pin in pins { + stdin_payload.push_str("delete "); + stdin_payload.push_str(&pin.ref_name); + stdin_payload.push(' '); + stdin_payload.push_str(&pin.tree.0); + stdin_payload.push('\n'); + } + + let mut child = Command::new("git") + .args(["update-ref", "--no-deref", "--stdin"]) + .current_dir(&self.repository_root) + .env("GIT_DIR", &self.git_dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| { + format!( + "Failed to run git update-ref --no-deref --stdin in '{}'", + self.repository_root.display() + ) + })?; + + child + .stdin + .take() + .ok_or_else(|| anyhow!("Failed to open stdin for git update-ref --no-deref --stdin"))? + .write_all(stdin_payload.as_bytes()) + .with_context(|| { + "Failed to write the delete transaction to git update-ref --no-deref --stdin" + })?; + + let output = child + .wait_with_output() + .with_context(|| "Failed to wait for git update-ref --no-deref --stdin")?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let detail = if stderr.is_empty() { stdout } else { stderr }; + return Err(anyhow!( + "git update-ref --no-deref --stdin failed: {detail}" + )); + } + + Ok(()) + } + + /// Fail closed unless every supplied pin is still exactly the direct ref + /// that was inventoried: present, a direct (non-symbolic) ref, targeting a + /// tree, and pointing at the recorded SHA. Re-inventoried in a single + /// `git for-each-ref` over the exact ref names, so no enumeration order is + /// relied on. This closes the common inventory→delete race cleanly; the + /// residual sub-transaction race is still contained by `--no-deref` plus + /// the per-`delete` old-value condition, which together cannot follow a + /// symbolic ref or mutate a ref the caller did not name. + fn assert_pins_are_unchanged_direct_refs(&self, pins: &[PinnedRef]) -> Result<()> { + let mut args: Vec<&str> = vec!["for-each-ref", FOR_EACH_REF_PIN_FORMAT]; + args.extend(pins.iter().map(|pin| pin.ref_name.as_str())); + let raw = self.run_git(&args, None)?; + + let current: Vec<[&str; 4]> = raw + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(|line| { + let fields: Vec<&str> = line.split('\0').collect(); + <[&str; 4]>::try_from(fields.as_slice()) + .map_err(|_| anyhow!("git for-each-ref emitted an unparseable line: '{line}'")) + }) + .collect::>()?; + + for pin in pins { + let Some(entry) = current.iter().find(|entry| entry[0] == pin.ref_name) else { + return Err(anyhow!( + "pin ref '{}' no longer exists; refusing to delete stale inventory", + pin.ref_name + )); + }; + let [_, object_name, object_type, symref] = *entry; + + if !symref.is_empty() { + return Err(anyhow!( + "pin ref '{}' is now a symbolic ref pointing at '{symref}'; mutation-cursor \ + pins must be direct refs, refusing to delete", + pin.ref_name + )); + } + if object_type != "tree" { + return Err(anyhow!( + "pin ref '{}' now targets a {object_type} object, not a tree; refusing to \ + delete", + pin.ref_name + )); + } + if object_name != pin.tree.0 { + return Err(anyhow!( + "pin ref '{}' now points at {object_name}, not the inventoried {}; refusing \ + to delete", + pin.ref_name, + pin.tree.0 + )); + } + } + + Ok(()) + } + fn head_exists(&self) -> Result { let output = Command::new("git") .args(["rev-parse", "--verify", "--quiet", "HEAD"]) @@ -125,8 +316,108 @@ impl GitSnapshotService { } } +/// One SCE-owned snapshot pin: a ref under +/// `refs/sce/mutation-cursor//` and the tree object it protects. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PinnedRef { + pub ref_name: String, + pub tree: TreeId, +} + +/// Why a worktree's pin inventory could not be produced. +#[derive(Debug)] +pub enum PinInventoryError { + /// `git for-each-ref` itself failed to execute or exited non-zero. + Git(anyhow::Error), + /// A ref under the SCE namespace is not shaped like a `pin_tree` output: a + /// symbolic ref, a non-tree target, a name/target SHA mismatch, an + /// unparseable `for-each-ref` line, or an unexpected extra path segment. + /// `reason` carries the specific discriminant for tests and `Display`. + MalformedRef { ref_name: String, reason: String }, +} + +impl std::fmt::Display for PinInventoryError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PinInventoryError::Git(source) => write!(f, "{source}"), + PinInventoryError::MalformedRef { ref_name, reason } => write!( + f, + "Malformed ref '{ref_name}' in the mutation-cursor snapshot namespace: {reason}" + ), + } + } +} + +impl std::error::Error for PinInventoryError {} + +fn parse_pin_line(line: &str, prefix: &str) -> std::result::Result { + let fields: Vec<&str> = line.split('\0').collect(); + let [ref_name, object_name, object_type, symref] = fields.as_slice() else { + return Err(PinInventoryError::MalformedRef { + ref_name: fields + .first() + .map_or_else(|| line.to_string(), |field| (*field).to_string()), + reason: format!( + "git for-each-ref line did not have exactly four NUL-separated fields: '{line}'" + ), + }); + }; + + if !symref.is_empty() { + return Err(PinInventoryError::MalformedRef { + ref_name: (*ref_name).to_string(), + reason: format!( + "ref is a symbolic ref pointing at '{symref}'; mutation-cursor pins must be direct \ + refs to a tree object, and a symbolic ref inside the namespace is rejected rather \ + than followed" + ), + }); + } + + if *object_type != "tree" { + return Err(PinInventoryError::MalformedRef { + ref_name: (*ref_name).to_string(), + reason: format!("ref target is a {object_type} object, not a tree"), + }); + } + + let Some(suffix) = ref_name.strip_prefix(prefix) else { + return Err(PinInventoryError::MalformedRef { + ref_name: (*ref_name).to_string(), + reason: format!("ref name is not under the expected prefix '{prefix}'"), + }); + }; + + if suffix.is_empty() || suffix.contains('/') { + return Err(PinInventoryError::MalformedRef { + ref_name: (*ref_name).to_string(), + reason: format!( + "ref name has an unexpected path segment after the worktree prefix: '{suffix}'" + ), + }); + } + + if suffix != *object_name { + return Err(PinInventoryError::MalformedRef { + ref_name: (*ref_name).to_string(), + reason: format!( + "ref name suffix '{suffix}' disagrees with its target tree SHA '{object_name}'" + ), + }); + } + + Ok(PinnedRef { + ref_name: (*ref_name).to_string(), + tree: TreeId((*object_name).to_string()), + }) +} + +fn pin_ref_prefix(worktree_id: &WorktreeId) -> String { + format!("{REF_NAMESPACE}/{}/", worktree_id.0) +} + fn pin_ref_name(worktree_id: &WorktreeId, tree: &TreeId) -> String { - format!("{REF_NAMESPACE}/{}/{}", worktree_id.0, tree.0) + format!("{}{}", pin_ref_prefix(worktree_id), tree.0) } fn resolve_git_dir(repository_root: &Path) -> Result { @@ -184,18 +475,31 @@ impl Drop for TempIndexGuard { #[cfg(test)] mod tests { - use std::sync::atomic::{AtomicU64, Ordering}; - use super::*; - static NEXT_TEST_REPO_ID: AtomicU64 = AtomicU64::new(0); + struct TestRepo { + _temp_dir: tempfile::TempDir, + root: PathBuf, + } + + impl TestRepo { + fn root(&self) -> &Path { + &self.root + } + } + + fn test_repo(label: &str) -> TestRepo { + let temp_dir = tempfile::Builder::new() + .prefix(&format!("sce-git-snapshot-{label}-")) + .tempdir() + .expect("test repository temp directory should be created"); - fn unique_test_repo(label: &str) -> PathBuf { - let id = NEXT_TEST_REPO_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-git-snapshot-{label}-{}-{id}", - std::process::id() - )) + let root = temp_dir.path().to_path_buf(); + + TestRepo { + _temp_dir: temp_dir, + root, + } } fn init_repo(repo_root: &Path) { @@ -224,17 +528,14 @@ mod tests { run(repo_root, &["commit", "--quiet", "-m", message]); } - fn remove_test_repo(repo_root: &Path) { - let _ = std::fs::remove_dir_all(repo_root); - } - fn worktree_id() -> WorktreeId { WorktreeId("test-worktree".to_string()) } #[test] fn capture_preserves_real_index_and_working_tree_state() { - let repo_root = unique_test_repo("preserves-index"); + let repo = test_repo("preserves-index"); + let repo_root = repo.root().to_path_buf(); init_repo(&repo_root); std::fs::write(repo_root.join("committed.txt"), b"original\n") .expect("committed file should be writable"); @@ -271,13 +572,12 @@ mod tests { assert!(ls_tree.contains("untracked.txt")); let committed_blob = run(&repo_root, &["show", &format!("{}:committed.txt", tree.0)]); assert_eq!(committed_blob, "modified\n"); - - remove_test_repo(&repo_root); } #[test] fn capture_excludes_ignored_files() { - let repo_root = unique_test_repo("ignored-files"); + let repo = test_repo("ignored-files"); + let repo_root = repo.root().to_path_buf(); init_repo(&repo_root); std::fs::write(repo_root.join(".gitignore"), b"ignored.txt\n") .expect(".gitignore should be writable"); @@ -292,13 +592,12 @@ mod tests { let ls_tree = run(&repo_root, &["ls-tree", "-r", "--name-only", &tree.0]); assert!(!ls_tree.contains("ignored.txt")); - - remove_test_repo(&repo_root); } #[test] fn capture_reflects_deletion_of_a_committed_file() { - let repo_root = unique_test_repo("deletion"); + let repo = test_repo("deletion"); + let repo_root = repo.root().to_path_buf(); init_repo(&repo_root); std::fs::write(repo_root.join("to-delete.txt"), b"will be removed\n") .expect("file should be writable"); @@ -312,13 +611,12 @@ mod tests { let ls_tree = run(&repo_root, &["ls-tree", "-r", "--name-only", &tree.0]); assert!(!ls_tree.contains("to-delete.txt")); - - remove_test_repo(&repo_root); } #[test] fn capture_on_unborn_head_with_a_file_produces_a_valid_tree() { - let repo_root = unique_test_repo("unborn-head-with-file"); + let repo = test_repo("unborn-head-with-file"); + let repo_root = repo.root().to_path_buf(); init_repo(&repo_root); std::fs::write(repo_root.join("untracked.txt"), b"before any commit\n") .expect("untracked file should be writable"); @@ -340,13 +638,12 @@ mod tests { let ls_tree = run(&repo_root, &["ls-tree", "-r", "--name-only", &tree.0]); assert!(ls_tree.contains("untracked.txt")); - - remove_test_repo(&repo_root); } #[test] fn an_unexpected_head_probe_failure_propagates_instead_of_using_read_tree_empty() { - let repo_root = unique_test_repo("head-probe-failure"); + let repo = test_repo("head-probe-failure"); + let repo_root = repo.root().to_path_buf(); init_repo(&repo_root); std::fs::write(repo_root.join("file.txt"), b"content\n").expect("file should be writable"); commit_all(&repo_root, "initial commit"); @@ -361,8 +658,6 @@ mod tests { result.is_err(), "an unexpected HEAD-probe failure must surface as an error, not a false empty-baseline capture" ); - - remove_test_repo(&repo_root); } fn relative_path_from(base: &Path, target: &Path) -> PathBuf { @@ -387,7 +682,8 @@ mod tests { #[test] fn resolves_an_absolute_git_dir_from_a_relative_repository_root() { - let repo_root_abs = unique_test_repo("relative-root"); + let repo = test_repo("relative-root"); + let repo_root_abs = repo.root().to_path_buf(); init_repo(&repo_root_abs); std::fs::write(repo_root_abs.join("file.txt"), b"relative content\n") .expect("file should be writable"); @@ -422,13 +718,12 @@ mod tests { .diff_trees(&tree, &tree) .expect("diff should succeed with a relative repository root"); assert!(diff.is_empty()); - - remove_test_repo(&repo_root_abs); } #[test] fn capture_on_unborn_head_with_no_files_produces_an_empty_tree() { - let repo_root = unique_test_repo("unborn-head-no-files"); + let repo = test_repo("unborn-head-no-files"); + let repo_root = repo.root().to_path_buf(); init_repo(&repo_root); let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); @@ -438,13 +733,12 @@ mod tests { let ls_tree = run(&repo_root, &["ls-tree", "-r", "--name-only", &tree.0]); assert_eq!(ls_tree.trim(), ""); - - remove_test_repo(&repo_root); } #[test] fn snapshot_survives_a_fresh_process_and_temp_index_deletion() { - let repo_root = unique_test_repo("survives-process-exit"); + let repo = test_repo("survives-process-exit"); + let repo_root = repo.root().to_path_buf(); init_repo(&repo_root); std::fs::write(repo_root.join("file.txt"), b"content\n").expect("file should be writable"); @@ -459,13 +753,12 @@ mod tests { let resolved = run(&repo_root, &["cat-file", "-t", &tree.0]); assert_eq!(resolved.trim(), "tree"); - - remove_test_repo(&repo_root); } #[test] fn pinned_snapshot_survives_git_gc_prune_now() { - let repo_root = unique_test_repo("gc-prune-now"); + let repo = test_repo("gc-prune-now"); + let repo_root = repo.root().to_path_buf(); init_repo(&repo_root); std::fs::write(repo_root.join("pinned.txt"), b"pinned content\n") .expect("file should be writable"); @@ -498,13 +791,12 @@ mod tests { !unpinned_probe.status.success(), "an unpinned, unreachable tree should be reclaimed by git gc --prune=now" ); - - remove_test_repo(&repo_root); } #[test] fn pinned_snapshot_survives_git_prune_expire_now() { - let repo_root = unique_test_repo("prune-expire-now"); + let repo = test_repo("prune-expire-now"); + let repo_root = repo.root().to_path_buf(); init_repo(&repo_root); std::fs::write(repo_root.join("pinned.txt"), b"pinned content\n") .expect("file should be writable"); @@ -537,13 +829,12 @@ mod tests { !unpinned_probe.status.success(), "an unpinned, unreachable tree should be reclaimed by git prune --expire=now" ); - - remove_test_repo(&repo_root); } #[test] fn pin_tree_is_idempotent_for_the_same_worktree_and_tree() { - let repo_root = unique_test_repo("pin-idempotent"); + let repo = test_repo("pin-idempotent"); + let repo_root = repo.root().to_path_buf(); init_repo(&repo_root); std::fs::write(repo_root.join("file.txt"), b"content\n").expect("file should be writable"); @@ -560,13 +851,12 @@ mod tests { let ref_name = pin_ref_name(&worktree_id(), &tree); let resolved = run(&repo_root, &["rev-parse", &ref_name]); assert_eq!(resolved.trim(), tree.0); - - remove_test_repo(&repo_root); } #[test] fn diff_trees_returns_parseable_git_diff_output() { - let repo_root = unique_test_repo("diff-trees"); + let repo = test_repo("diff-trees"); + let repo_root = repo.root().to_path_buf(); init_repo(&repo_root); std::fs::write(repo_root.join("file.txt"), b"before\n").expect("file should be writable"); let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); @@ -581,13 +871,12 @@ mod tests { assert!(diff.contains("file.txt")); assert!(diff.contains("-before")); assert!(diff.contains("+after")); - - remove_test_repo(&repo_root); } #[test] fn capture_and_pin_work_against_a_sha256_repository_when_supported() { - let repo_root = unique_test_repo("sha256"); + let repo = test_repo("sha256"); + let repo_root = repo.root().to_path_buf(); std::fs::create_dir_all(&repo_root).expect("repo root should be created"); let init = Command::new("git") .args(["init", "--quiet", "--object-format=sha256"]) @@ -595,7 +884,6 @@ mod tests { .output() .expect("git init should spawn"); if !init.status.success() { - remove_test_repo(&repo_root); return; } run(&repo_root, &["config", "user.email", "test@example.com"]); @@ -612,7 +900,383 @@ mod tests { let ls_tree = run(&repo_root, &["ls-tree", "-r", "--name-only", &tree.0]); assert!(ls_tree.contains("file.txt")); + } + + fn other_worktree_id() -> WorktreeId { + WorktreeId("other-worktree".to_string()) + } - remove_test_repo(&repo_root); + fn capture_with_file( + service: &GitSnapshotService, + repo_root: &Path, + name: &str, + body: &[u8], + ) -> TreeId { + std::fs::write(repo_root.join(name), body).expect("file should be writable"); + service.capture_tree().expect("capture should succeed") + } + + fn ref_target(repo_root: &Path, ref_name: &str) -> String { + run(repo_root, &["rev-parse", "--verify", ref_name]) + .trim() + .to_string() + } + + fn ref_exists(repo_root: &Path, ref_name: &str) -> bool { + Command::new("git") + .args(["rev-parse", "--verify", "--quiet", ref_name]) + .current_dir(repo_root) + .output() + .expect("git rev-parse should spawn") + .status + .success() + } + + #[test] + fn list_pins_returns_only_the_target_worktree_prefix() { + let repo = test_repo("list-pins-scoped"); + let repo_root = repo.root().to_path_buf(); + init_repo(&repo_root); + let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); + + let tree_a = capture_with_file(&service, &repo_root, "a.txt", b"a\n"); + let tree_b = capture_with_file(&service, &repo_root, "b.txt", b"b\n"); + + service + .pin_tree(&worktree_id(), &tree_a) + .expect("pin should succeed"); + service + .pin_tree(&worktree_id(), &tree_b) + .expect("pin should succeed"); + service + .pin_tree(&other_worktree_id(), &tree_a) + .expect("pin should succeed"); + + let pins = service + .list_pins(&worktree_id()) + .expect("inventory should succeed"); + assert_eq!(pins.len(), 2, "only the target worktree's pins are listed"); + + let mut trees: Vec = pins.iter().map(|pin| pin.tree.0.clone()).collect(); + trees.sort(); + let mut expected = vec![tree_a.0.clone(), tree_b.0.clone()]; + expected.sort(); + assert_eq!(trees, expected); + + let prefix = format!("{REF_NAMESPACE}/{}/", worktree_id().0); + for pin in &pins { + assert!(pin.ref_name.starts_with(&prefix)); + assert_eq!(pin.ref_name, format!("{prefix}{}", pin.tree.0)); + } + + assert!(ref_exists( + &repo_root, + &format!("{REF_NAMESPACE}/{}/{}", other_worktree_id().0, tree_a.0) + )); + } + + #[test] + fn list_pins_is_empty_when_the_worktree_has_no_pins() { + let repo = test_repo("list-pins-empty"); + let repo_root = repo.root().to_path_buf(); + init_repo(&repo_root); + let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); + + let pins = service + .list_pins(&worktree_id()) + .expect("inventory should succeed"); + assert!(pins.is_empty()); + } + + #[test] + fn list_pins_rejects_a_ref_whose_target_is_not_a_tree() { + let repo = test_repo("list-pins-non-tree"); + let repo_root = repo.root().to_path_buf(); + init_repo(&repo_root); + std::fs::write(repo_root.join("file.txt"), b"content\n").expect("file should be writable"); + commit_all(&repo_root, "initial commit"); + let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); + + let head = run(&repo_root, &["rev-parse", "HEAD"]).trim().to_string(); + let ref_name = format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, head); + run(&repo_root, &["update-ref", &ref_name, &head]); + + match service.list_pins(&worktree_id()) { + Err(PinInventoryError::MalformedRef { + ref_name: rn, + reason, + }) => { + assert_eq!(rn, ref_name); + assert!( + reason.contains("commit"), + "reason names the wrong object type: {reason}" + ); + } + other => panic!("expected MalformedRef for a non-tree target, got {other:?}"), + } + } + + #[test] + fn list_pins_rejects_a_ref_whose_name_disagrees_with_its_target() { + let repo = test_repo("list-pins-name-mismatch"); + let repo_root = repo.root().to_path_buf(); + init_repo(&repo_root); + let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); + + let tree_a = capture_with_file(&service, &repo_root, "a.txt", b"a\n"); + let tree_b = capture_with_file(&service, &repo_root, "b.txt", b"b\n"); + + let ref_name = format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_a.0); + run(&repo_root, &["update-ref", &ref_name, &tree_b.0]); + + match service.list_pins(&worktree_id()) { + Err(PinInventoryError::MalformedRef { + ref_name: rn, + reason, + }) => { + assert_eq!(rn, ref_name); + assert!(reason.contains("disagrees"), "unexpected reason: {reason}"); + } + other => panic!("expected MalformedRef for a name/target mismatch, got {other:?}"), + } + } + + #[test] + fn list_pins_rejects_a_ref_with_an_unexpected_extra_path_segment() { + let repo = test_repo("list-pins-extra-segment"); + let repo_root = repo.root().to_path_buf(); + init_repo(&repo_root); + let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); + + let tree_a = capture_with_file(&service, &repo_root, "a.txt", b"a\n"); + let ref_name = format!("{REF_NAMESPACE}/{}/nested/{}", worktree_id().0, tree_a.0); + run(&repo_root, &["update-ref", &ref_name, &tree_a.0]); + + match service.list_pins(&worktree_id()) { + Err(PinInventoryError::MalformedRef { reason, .. }) => { + assert!( + reason.contains("path segment"), + "unexpected reason: {reason}" + ); + } + other => panic!("expected MalformedRef for an extra path segment, got {other:?}"), + } + } + + #[test] + fn list_pins_reports_a_for_each_ref_execution_failure_as_the_git_variant() { + let repo = test_repo("list-pins-git-failure"); + let repo_root = repo.root().to_path_buf(); + init_repo(&repo_root); + let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); + + std::fs::remove_dir_all(&service.git_dir).expect("git-dir should be removable"); + + match service.list_pins(&worktree_id()) { + Err(PinInventoryError::Git(_)) => {} + other => panic!( + "expected the Git variant for a for-each-ref execution failure, got {other:?}" + ), + } + } + + #[test] + fn delete_pins_removes_exactly_the_supplied_refs() { + let repo = test_repo("delete-pins-exact"); + let repo_root = repo.root().to_path_buf(); + init_repo(&repo_root); + let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); + + let tree_a = capture_with_file(&service, &repo_root, "a.txt", b"a\n"); + let tree_b = capture_with_file(&service, &repo_root, "b.txt", b"b\n"); + service + .pin_tree(&worktree_id(), &tree_a) + .expect("pin should succeed"); + service + .pin_tree(&worktree_id(), &tree_b) + .expect("pin should succeed"); + + let inventory = service + .list_pins(&worktree_id()) + .expect("inventory should succeed"); + let stale: Vec = inventory + .into_iter() + .filter(|pin| pin.tree == tree_a) + .collect(); + service.delete_pins(&stale).expect("delete should succeed"); + + let remaining = service + .list_pins(&worktree_id()) + .expect("inventory should succeed"); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].tree, tree_b); + assert!(!ref_exists( + &repo_root, + &format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_a.0) + )); + } + + #[test] + fn delete_pins_is_a_successful_noop_for_an_empty_slice() { + let repo = test_repo("delete-pins-empty"); + let repo_root = repo.root().to_path_buf(); + init_repo(&repo_root); + let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); + + let tree_a = capture_with_file(&service, &repo_root, "a.txt", b"a\n"); + service + .pin_tree(&worktree_id(), &tree_a) + .expect("pin should succeed"); + + service + .delete_pins(&[]) + .expect("empty delete should be a successful no-op"); + + assert!(ref_exists( + &repo_root, + &format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_a.0) + )); + } + + #[test] + fn delete_pins_atomically_aborts_when_a_ref_changes_after_preflight() { + let repo = test_repo("delete-pins-atomic-abort"); + let repo_root = repo.root().to_path_buf(); + init_repo(&repo_root); + let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); + + let tree_a = capture_with_file(&service, &repo_root, "a.txt", b"a\n"); + let tree_b = capture_with_file(&service, &repo_root, "b.txt", b"b\n"); + let tree_c = capture_with_file(&service, &repo_root, "c.txt", b"c\n"); + service + .pin_tree(&worktree_id(), &tree_a) + .expect("pin should succeed"); + service + .pin_tree(&worktree_id(), &tree_b) + .expect("pin should succeed"); + + let ref_a = format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_a.0); + let ref_b = format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_b.0); + + let valid_ref = PinnedRef { + ref_name: ref_a.clone(), + tree: tree_a.clone(), + }; + let mismatched_ref = PinnedRef { + ref_name: ref_b.clone(), + tree: tree_b.clone(), + }; + + let pins = [valid_ref, mismatched_ref]; + + let result = service.delete_pins_inner(&pins, || { + run(&repo_root, &["update-ref", &ref_b, &tree_c.0]); + }); + + result.expect_err( + "the transaction is issued after preflight; git's per-delete expected-old-value \ + check on the second ref must abort the whole batch", + ); + + assert!( + ref_exists(&repo_root, &ref_a), + "the first (valid) delete must not have been applied — the batch is all-or-nothing" + ); + assert_eq!(ref_target(&repo_root, &ref_a), tree_a.0); + + assert!( + ref_exists(&repo_root, &ref_b), + "the mismatched ref still exists" + ); + assert_eq!( + ref_target(&repo_root, &ref_b), + tree_c.0, + "the mismatched ref keeps the value it was given after preflight" + ); + } + + #[test] + fn list_pins_rejects_a_symbolic_ref_inside_the_mutation_cursor_namespace() { + let repo = test_repo("list-pins-symref"); + let repo_root = repo.root().to_path_buf(); + init_repo(&repo_root); + let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); + + let tree_t = capture_with_file(&service, &repo_root, "t.txt", b"t\n"); + + let b_ref = format!("{REF_NAMESPACE}/{}/{}", other_worktree_id().0, tree_t.0); + let a_ref = format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_t.0); + run(&repo_root, &["update-ref", &b_ref, &tree_t.0]); + run(&repo_root, &["symbolic-ref", &a_ref, &b_ref]); + + match service.list_pins(&worktree_id()) { + Err(PinInventoryError::MalformedRef { + ref_name: rn, + reason, + }) => { + assert_eq!(rn, a_ref); + assert!( + reason.contains("symbolic ref"), + "unexpected reason: {reason}" + ); + } + other => { + panic!("expected MalformedRef for a symbolic ref in the namespace, got {other:?}") + } + } + + assert!(ref_exists(&repo_root, &b_ref)); + assert_eq!(ref_target(&repo_root, &b_ref), tree_t.0); + assert_eq!( + run(&repo_root, &["symbolic-ref", &a_ref]).trim(), + b_ref, + "A/T must still be the untouched symbolic ref" + ); + } + + #[test] + fn delete_pins_refuses_to_act_when_an_inventoried_direct_ref_became_a_symbolic_ref() { + let repo = test_repo("delete-pins-symref-race"); + let repo_root = repo.root().to_path_buf(); + init_repo(&repo_root); + let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); + + let tree_t = capture_with_file(&service, &repo_root, "t.txt", b"t\n"); + let a_ref = format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_t.0); + let b_ref = format!("{REF_NAMESPACE}/{}/{}", other_worktree_id().0, tree_t.0); + + service + .pin_tree(&worktree_id(), &tree_t) + .expect("pin should succeed"); + let inventory = service + .list_pins(&worktree_id()) + .expect("inventory should succeed"); + assert_eq!( + inventory, + vec![PinnedRef { + ref_name: a_ref.clone(), + tree: tree_t.clone(), + }] + ); + + run(&repo_root, &["update-ref", &b_ref, &tree_t.0]); + run(&repo_root, &["update-ref", "-d", &a_ref]); + run(&repo_root, &["symbolic-ref", &a_ref, &b_ref]); + + let error = service + .delete_pins(&inventory) + .expect_err("delete must fail closed once an inventoried direct ref became a symref"); + assert!( + error.to_string().contains("symbolic ref"), + "unexpected error: {error}" + ); + + assert!(ref_exists(&repo_root, &b_ref)); + assert_eq!(ref_target(&repo_root, &b_ref), tree_t.0); + assert_eq!( + run(&repo_root, &["symbolic-ref", &a_ref]).trim(), + b_ref, + "A/T is not touched — delete_pins prefers failure over acting on a symref" + ); } } diff --git a/cli/src/services/mutation_trace/runtime/mod.rs b/cli/src/services/mutation_trace/runtime/mod.rs index 703058a6..2d88705c 100644 --- a/cli/src/services/mutation_trace/runtime/mod.rs +++ b/cli/src/services/mutation_trace/runtime/mod.rs @@ -1,6 +1,7 @@ mod coordinator; mod external_taint; mod git_snapshot; +mod ref_reconciliation; mod worktree_lock; #[cfg(test)] diff --git a/cli/src/services/mutation_trace/runtime/ref_reconciliation.rs b/cli/src/services/mutation_trace/runtime/ref_reconciliation.rs new file mode 100644 index 00000000..fc10094f --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_reconciliation.rs @@ -0,0 +1,753 @@ +use std::collections::BTreeSet; +use std::path::Path; +use std::time::Duration; + +use anyhow::Result; + +use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::checkout::{read_checkout_id, resolve_git_dir}; +use crate::services::mutation_trace::store::MutationTraceStore; +use crate::services::mutation_trace::types::{TreeId, WorktreeId}; + +use super::git_snapshot::{GitSnapshotService, PinInventoryError, PinnedRef}; +use super::worktree_lock::{acquire_inner, WorktreeLockError}; + +/// Bounded wait for the worktree's `WorktreeLock` before a reconciliation pass +/// gives up. Its value intentionally matches the coordinator's private +/// `WORKTREE_LOCK_TIMEOUT` but is deliberately **not** a shared abstraction: +/// there is no semantic reason the two timeouts must always stay identical, so +/// each module owns its own constant. +const RECONCILIATION_LOCK_TIMEOUT: Duration = Duration::from_secs(10); + +/// Outcome counts of one successful reconciliation pass. +/// +/// - `local_required` — the target worktree's own durable-root count +/// (`load_tree_roots(W).len()`), the left side of the local consistency +/// invariant. +/// - `retained` — `actual_W.len() - deleted`: pins left in place, whether +/// because the target worktree still needs their tree or because another +/// worktree in the repository durably does. +/// - `deleted` — pins actually removed (inventoried under `W`'s prefix, tree +/// absent from the repository-wide durable root set). +/// +/// `retained == local_required` is **not** an invariant — a pin retained only +/// because another worktree durably needs its tree counts toward `retained` +/// but not `local_required`. For `ReconciliationOutcome::Reconciled(report)` +/// the only relation that holds is `report.local_required <= report.retained`. +/// `ReconciliationOutcome::SkippedNoCheckoutIdentity` carries no report, so no +/// report invariant applies to it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReconciliationReport { + pub local_required: usize, + pub retained: usize, + pub deleted: usize, +} + +/// Outcome of one reconciliation pass: a real pass that ran, carrying its +/// [`ReconciliationReport`], versus a skip because no current checkout identity +/// could be derived (an `Ok`, never an `Err`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReconciliationOutcome { + Reconciled(ReconciliationReport), + SkippedNoCheckoutIdentity, +} + +/// Why a reconciliation pass could not complete. One variant per fallible step, +/// no `Other` catch-all — mirroring `CoordinateError`'s convention. Every +/// non-`Ok` outcome leaves the SCE ref namespace in a consistent state: either +/// untouched, or (only on `Ok`) with exactly the stale refs gone. +#[derive(Debug)] +pub enum ReconcileError { + /// `resolve_git_dir` failed. + GitDir(anyhow::Error), + /// The worktree's `WorktreeLock` could not be acquired (timeout or I/O). + Lock(WorktreeLockError), + /// `read_checkout_id` returned `Err` — a corrupt or unreadable checkout id, + /// which is **not** the same as an absent one (`Ok(None)` is a clean no-op). + CheckoutIdentity(anyhow::Error), + /// The caller-supplied `open_db` provider returned `Err`. This is a + /// reconciliation maintenance error only: it never arms + /// `ExternalTaintMarker` and never becomes + /// `CoordinateError::AgentTraceDbUnavailable`, because no mutation boundary + /// is being coordinated. + AgentTraceDbUnavailable(anyhow::Error), + /// `GitSnapshotService::new` failed. + SnapshotService(anyhow::Error), + /// `git for-each-ref` itself failed to execute or exited non-zero + /// (`PinInventoryError::Git`). + PinInventory(anyhow::Error), + /// A ref inside the SCE mutation-cursor namespace is not shaped like a + /// `pin_tree` output (`PinInventoryError::MalformedRef`). Reconciliation + /// deletes nothing. + MalformedPin { ref_name: String, reason: String }, + /// `load_tree_roots` / `load_all_tree_roots` failed (DB query error, + /// migration `003` absent, ...). + DurableRoots(anyhow::Error), + /// A durable root of the **target** worktree has no live pin — the local + /// consistency invariant is violated. Fail closed: nothing is deleted. + MissingRequiredPins { missing: Vec }, + /// The atomic `delete_pins` transaction failed (including a ref that + /// changed since inventory). Nothing is deleted. + DeleteTransaction(anyhow::Error), +} + +impl std::fmt::Display for ReconcileError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ReconcileError::Lock(source) => write!(f, "{source}"), + ReconcileError::MalformedPin { ref_name, reason } => write!( + f, + "Malformed ref '{ref_name}' in the mutation-cursor snapshot \ + namespace; reconciliation deleted nothing: {reason}" + ), + ReconcileError::MissingRequiredPins { missing } => write!( + f, + "{} durable root(s) of the target worktree have no snapshot pin; \ + reconciliation failed closed and deleted nothing: {missing:?}", + missing.len() + ), + ReconcileError::AgentTraceDbUnavailable(source) => { + write!(f, "Repository Agent Trace DB is unavailable: {source}") + } + ReconcileError::GitDir(source) + | ReconcileError::CheckoutIdentity(source) + | ReconcileError::SnapshotService(source) + | ReconcileError::PinInventory(source) + | ReconcileError::DurableRoots(source) + | ReconcileError::DeleteTransaction(source) => write!(f, "{source}"), + } + } +} + +impl std::error::Error for ReconcileError {} + +/// Reconcile one worktree's SCE snapshot pins: remove orphan / unreferenced +/// pins while retaining every tree any current or historical durable +/// mutation-cursor state in the repository still references. +/// +/// Module-private to `runtime`, exactly like `coordinate` — never re-exported +/// outside mutation-trace `runtime`. It is a one-line delegation to +/// [`reconcile_worktree_inner`] with a no-op lock-contention closure. +pub fn reconcile_worktree

( + repository_root: &Path, + open_db: P, +) -> std::result::Result +where + P: FnOnce() -> Result, +{ + reconcile_worktree_inner(repository_root, open_db, || {}) +} + +/// Body of [`reconcile_worktree`] with a deterministic test seam: +/// `on_lock_contention` fires once the moment the pass first observes the +/// `WorktreeLock` is already held by another owner (see +/// [`super::worktree_lock::acquire_inner`]). `pub(super)` keeps it reachable +/// from `runtime` and `runtime::tests` but invisible outside `runtime`. +/// +/// Every fallible step below runs entirely while holding the worktree's +/// `WorktreeLock`, which is the same lock file `coordinate()` holds across +/// `pin -> recovery -> prepare -> CAS -> marker clear -> return` — the mutual +/// exclusion that makes the pin -> DB-CAS race structurally impossible. +pub(super) fn reconcile_worktree_inner( + repository_root: &Path, + open_db: P, + on_lock_contention: F, +) -> std::result::Result +where + P: FnOnce() -> Result, + F: FnOnce(), +{ + let git_dir = resolve_git_dir(repository_root).map_err(ReconcileError::GitDir)?; + + let _lock = acquire_inner(&git_dir, RECONCILIATION_LOCK_TIMEOUT, on_lock_contention) + .map_err(ReconcileError::Lock)?; + + // The lock is held from here until this function returns. + let worktree_id = match read_checkout_id(&git_dir).map_err(ReconcileError::CheckoutIdentity)? { + Some(id) => WorktreeId(id), + // No current checkout identity to derive a `WorktreeId` and its owned + // `refs/sce/mutation-cursor//` prefix from — nothing to + // inventory, validate, or delete. Clean no-op; no identity is created. + None => { + return Ok(ReconciliationOutcome::SkippedNoCheckoutIdentity); + } + }; + + let db = open_db().map_err(ReconcileError::AgentTraceDbUnavailable)?; + + let snapshot = + GitSnapshotService::new(repository_root).map_err(ReconcileError::SnapshotService)?; + + // Inventory the worktree's pins first, so every durable-root read that + // follows is compared against a fixed observation of the namespace. + let actual = snapshot + .list_pins(&worktree_id) + .map_err(|error| match error { + PinInventoryError::Git(source) => ReconcileError::PinInventory(source), + PinInventoryError::MalformedRef { ref_name, reason } => { + ReconcileError::MalformedPin { ref_name, reason } + } + })?; + let pinned_trees: BTreeSet = actual.iter().map(|pin| pin.tree.clone()).collect(); + + let store = MutationTraceStore::new(&db); + + // Local consistency invariant (a strictly per-worktree check): every tree + // the target worktree's own durable evidence references must still have a + // live pin, or the pass fails closed and deletes nothing. + let required_local = store + .load_tree_roots(&worktree_id) + .map_err(ReconcileError::DurableRoots)?; + let missing_local: Vec = required_local.difference(&pinned_trees).cloned().collect(); + if !missing_local.is_empty() { + return Err(ReconcileError::MissingRequiredPins { + missing: missing_local, + }); + } + + // Deletion safety invariant: an owned ref is removed only when its tree is + // outside the durable root set of **every** worktree in the repository — + // linked worktrees share one object database, so an A-owned ref may be the + // last SCE ref protecting a tree that only worktree B durably requires. + let required_repository = store + .load_all_tree_roots() + .map_err(ReconcileError::DurableRoots)?; + let stale: Vec = actual + .iter() + .filter(|pin| !required_repository.contains(&pin.tree)) + .cloned() + .collect(); + + if !stale.is_empty() { + snapshot + .delete_pins(&stale) + .map_err(ReconcileError::DeleteTransaction)?; + } + + Ok(ReconciliationOutcome::Reconciled(ReconciliationReport { + local_required: required_local.len(), + retained: actual.len() - stale.len(), + deleted: stale.len(), + })) +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + use std::process::Command; + + use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; + use crate::services::checkout::get_or_create_checkout_id; + use crate::services::mutation_trace::store::encode_revision; + + use super::*; + + const NAMESPACE: &str = "refs/sce/mutation-cursor"; + + struct Fixture { + _temp_dir: tempfile::TempDir, + repo_root: PathBuf, + db_path: PathBuf, + worktree_id: WorktreeId, + } + + fn expect_reconciled( + outcome: std::result::Result, + ) -> ReconciliationReport { + match outcome.expect("reconciliation should succeed") { + ReconciliationOutcome::Reconciled(report) => report, + ReconciliationOutcome::SkippedNoCheckoutIdentity => { + panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") + } + } + } + + fn git(dir: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .expect("git command should spawn"); + assert!( + output.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).expect("git output should be valid UTF-8") + } + + fn fixture(label: &str) -> Fixture { + let temp_dir = tempfile::Builder::new() + .prefix(&format!("sce-ref-reconciliation-{label}-")) + .tempdir() + .expect("test temp directory should be created"); + + let repo_root = temp_dir.path().join("repo"); + std::fs::create_dir_all(&repo_root).expect("repo root should be created"); + git(&repo_root, &["init", "--quiet"]); + git(&repo_root, &["config", "user.email", "test@example.com"]); + git(&repo_root, &["config", "user.name", "Test"]); + git( + &repo_root, + &["commit", "--allow-empty", "--quiet", "-m", "init"], + ); + + let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); + let checkout_id = get_or_create_checkout_id(&git_dir).expect("checkout id should resolve"); + + // The DB lives beside the worktree, never inside it, so it can never + // perturb a captured tree. + let db_path = temp_dir.path().join("agent-trace.db"); + RepositoryAgentTraceDb::new_at(&db_path).expect("repository schema DB should open"); + + Fixture { + _temp_dir: temp_dir, + repo_root, + db_path, + worktree_id: WorktreeId(checkout_id), + } + } + + impl Fixture { + fn open_db(&self) -> Result { + RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&self.db_path) + } + + fn db(&self) -> RepositoryAgentTraceDb { + self.open_db().expect("repository DB should reopen") + } + + fn snapshot(&self) -> GitSnapshotService { + GitSnapshotService::new(&self.repo_root).expect("snapshot service should construct") + } + + /// Write `contents` to `file` in the worktree and capture the resulting + /// tree, so successive distinct contents yield distinct tree SHAs. + fn capture_after_writing(&self, file: &str, contents: &str) -> TreeId { + std::fs::write(self.repo_root.join(file), contents) + .expect("worktree file should write"); + self.snapshot() + .capture_tree() + .expect("capture should succeed") + } + + fn pin(&self, tree: &TreeId) { + self.snapshot() + .pin_tree(&self.worktree_id, tree) + .expect("pin should succeed"); + } + + fn pin_for(&self, worktree_id: &WorktreeId, tree: &TreeId) { + self.snapshot() + .pin_tree(worktree_id, tree) + .expect("pin should succeed"); + } + + fn reconcile(&self) -> std::result::Result { + reconcile_worktree(&self.repo_root, || self.open_db()) + } + + fn owned_ref(&self, tree: &TreeId) -> String { + format!("{NAMESPACE}/{}/{}", self.worktree_id.0, tree.0) + } + + fn ref_exists(&self, ref_name: &str) -> bool { + Command::new("git") + .args(["show-ref", "--verify", "--quiet", ref_name]) + .current_dir(&self.repo_root) + .status() + .expect("git show-ref should spawn") + .success() + } + + fn ref_representation(&self, ref_name: &str) -> String { + git( + &self.repo_root, + &[ + "for-each-ref", + "--format=%(refname)%00%(objectname)%00%(objecttype)%00%(symref)", + ref_name, + ], + ) + } + + fn object_type(&self, sha: &str) -> Option { + let output = Command::new("git") + .args(["cat-file", "-t", sha]) + .current_dir(&self.repo_root) + .output() + .expect("git cat-file should spawn"); + if output.status.success() { + Some( + String::from_utf8(output.stdout) + .expect("git cat-file output should be UTF-8") + .trim() + .to_string(), + ) + } else { + None + } + } + } + + fn seed_worktree_cursor( + db: &RepositoryAgentTraceDb, + worktree_id: &str, + revision: u64, + cursor_tree: &str, + ) { + db.execute( + "INSERT INTO mutation_trace_worktrees + (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) + VALUES (?1, ?2, ?3, 0, 'healthy', 0)", + ( + worktree_id, + cursor_tree, + encode_revision(revision).as_slice(), + ), + ) + .expect("worktree row insert should succeed"); + } + + fn seed_event( + db: &RepositoryAgentTraceDb, + worktree_id: &str, + revision: u64, + before_tree: &str, + after_tree: &str, + ) { + db.execute( + "INSERT INTO mutation_trace_events + (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, + attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, + boundary_event_id) + VALUES (?1, ?2, ?3, ?4, 0, 'healthy', 'ineligible_unscoped', NULL, 'flush', NULL, NULL)", + ( + worktree_id, + encode_revision(revision).as_slice(), + before_tree, + after_tree, + ), + ) + .expect("event row insert should succeed"); + } + + #[test] + fn orphan_pin_with_a_worktree_row_is_deleted() { + let fx = fixture("orphan-with-row"); + let cursor = fx.capture_after_writing("a.txt", "cursor\n"); + let orphan = fx.capture_after_writing("a.txt", "orphan\n"); + fx.pin(&cursor); + fx.pin(&orphan); + seed_worktree_cursor(&fx.db(), &fx.worktree_id.0, 1, &cursor.0); + + let report = expect_reconciled(fx.reconcile()); + + assert_eq!( + report, + ReconciliationReport { + local_required: 1, + retained: 1, + deleted: 1, + } + ); + assert!(fx.ref_exists(&fx.owned_ref(&cursor)), "cursor pin retained"); + assert!(!fx.ref_exists(&fx.owned_ref(&orphan)), "orphan pin deleted"); + } + + #[test] + fn orphan_pin_with_no_worktree_row_is_deleted() { + let fx = fixture("orphan-no-row"); + let orphan = fx.capture_after_writing("a.txt", "orphan\n"); + fx.pin(&orphan); + + let report = expect_reconciled(fx.reconcile()); + + assert_eq!( + report, + ReconciliationReport { + local_required: 0, + retained: 0, + deleted: 1, + } + ); + assert!(!fx.ref_exists(&fx.owned_ref(&orphan)), "orphan pin deleted"); + } + + #[test] + fn current_cursor_pin_is_retained_without_a_referencing_event() { + let fx = fixture("cursor-no-event"); + let cursor = fx.capture_after_writing("a.txt", "cursor\n"); + fx.pin(&cursor); + seed_worktree_cursor(&fx.db(), &fx.worktree_id.0, 3, &cursor.0); + + let report = expect_reconciled(fx.reconcile()); + + assert_eq!( + report, + ReconciliationReport { + local_required: 1, + retained: 1, + deleted: 0, + } + ); + assert!( + fx.ref_exists(&fx.owned_ref(&cursor)), + "current cursor pin retained" + ); + } + + #[test] + fn historical_event_before_and_after_pins_are_retained_after_the_cursor_advances() { + let fx = fixture("historical-retention"); + let tree_a = fx.capture_after_writing("a.txt", "A\n"); + let tree_b = fx.capture_after_writing("a.txt", "B\n"); + let tree_c = fx.capture_after_writing("a.txt", "C\n"); + let tree_d = fx.capture_after_writing("a.txt", "D\n"); + for tree in [&tree_a, &tree_b, &tree_c, &tree_d] { + fx.pin(tree); + } + + let db = fx.db(); + seed_worktree_cursor(&db, &fx.worktree_id.0, 3, &tree_d.0); + seed_event(&db, &fx.worktree_id.0, 1, &tree_a.0, &tree_b.0); + seed_event(&db, &fx.worktree_id.0, 2, &tree_b.0, &tree_c.0); + seed_event(&db, &fx.worktree_id.0, 3, &tree_c.0, &tree_d.0); + + let report = expect_reconciled(fx.reconcile()); + + assert_eq!( + report, + ReconciliationReport { + local_required: 4, + retained: 4, + deleted: 0, + } + ); + for tree in [&tree_a, &tree_b, &tree_c, &tree_d] { + assert!( + fx.ref_exists(&fx.owned_ref(tree)), + "historical tree {} pin retained", + tree.0 + ); + } + } + + #[test] + fn a_pin_another_worktree_durably_requires_is_retained() { + let fx = fixture("cross-worktree-retention"); + let shared = fx.capture_after_writing("a.txt", "shared\n"); + let orphan = fx.capture_after_writing("a.txt", "orphan\n"); + fx.pin(&shared); + fx.pin(&orphan); + + // Another worktree in the same repository durably references `shared`; + // this worktree does not. + seed_worktree_cursor(&fx.db(), "other-worktree", 1, &shared.0); + + let report = expect_reconciled(fx.reconcile()); + + assert_eq!( + report, + ReconciliationReport { + local_required: 0, + retained: 1, + deleted: 1, + }, + "a pin another worktree durably needs is retained even though it is \ + not a local root; retained exceeds local_required" + ); + assert!( + fx.ref_exists(&fx.owned_ref(&shared)), + "the repository-wide durable tree pin is retained" + ); + assert!( + !fx.ref_exists(&fx.owned_ref(&orphan)), + "the orphan pin is deleted" + ); + assert_eq!( + fx.object_type(&shared.0).as_deref(), + Some("tree"), + "the retained ref keeps the shared tree resolvable" + ); + } + + #[test] + fn a_missing_required_pin_fails_closed_and_deletes_nothing() { + let fx = fixture("missing-required-pin"); + let tree_a = fx.capture_after_writing("a.txt", "A\n"); + let tree_b = fx.capture_after_writing("a.txt", "B\n"); + let tree_x = fx.capture_after_writing("a.txt", "X\n"); + // Local durable roots are {A, B}; only A and X are pinned (B has no pin). + fx.pin(&tree_a); + fx.pin(&tree_x); + + let db = fx.db(); + seed_worktree_cursor(&db, &fx.worktree_id.0, 1, &tree_b.0); + seed_event(&db, &fx.worktree_id.0, 1, &tree_a.0, &tree_b.0); + + let error = fx + .reconcile() + .expect_err("reconciliation should fail closed"); + + match error { + ReconcileError::MissingRequiredPins { missing } => { + assert_eq!( + missing, + vec![tree_b.clone()], + "the missing local root is named" + ); + } + other => panic!("expected MissingRequiredPins, got {other:?}"), + } + assert!( + fx.ref_exists(&fx.owned_ref(&tree_a)), + "A's pin is left in place" + ); + assert!( + fx.ref_exists(&fx.owned_ref(&tree_x)), + "X's pin is left in place" + ); + } + + #[test] + fn a_malformed_namespace_ref_fails_closed_and_deletes_nothing() { + let fx = fixture("malformed-ref"); + let cursor = fx.capture_after_writing("a.txt", "cursor\n"); + let orphan = fx.capture_after_writing("a.txt", "orphan\n"); + fx.pin(&cursor); + fx.pin(&orphan); + seed_worktree_cursor(&fx.db(), &fx.worktree_id.0, 1, &cursor.0); + + // A symbolic ref inside the SCE mutation-cursor namespace is malformed. + let symref = format!("{NAMESPACE}/{}/symbolic", fx.worktree_id.0); + git( + &fx.repo_root, + &["symbolic-ref", &symref, &fx.owned_ref(&cursor)], + ); + + let error = fx + .reconcile() + .expect_err("reconciliation should fail closed"); + + match error { + ReconcileError::MalformedPin { ref_name, .. } => { + assert_eq!(ref_name, symref, "the malformed ref is named"); + } + other => panic!("expected MalformedPin, got {other:?}"), + } + assert!( + fx.ref_exists(&fx.owned_ref(&cursor)), + "the cursor pin is untouched" + ); + assert!( + fx.ref_exists(&fx.owned_ref(&orphan)), + "the orphan pin is untouched" + ); + } + + #[test] + fn reconciliation_is_idempotent() { + let fx = fixture("idempotent"); + let cursor = fx.capture_after_writing("a.txt", "cursor\n"); + let orphan = fx.capture_after_writing("a.txt", "orphan\n"); + fx.pin(&cursor); + fx.pin(&orphan); + seed_worktree_cursor(&fx.db(), &fx.worktree_id.0, 1, &cursor.0); + + let first = expect_reconciled(fx.reconcile()); + assert_eq!( + first, + ReconciliationReport { + local_required: 1, + retained: 1, + deleted: 1, + } + ); + + let second = expect_reconciled(fx.reconcile()); + assert_eq!( + second, + ReconciliationReport { + local_required: 1, + retained: 1, + deleted: 0, + }, + "a second pass with no intervening change deletes nothing and \ + reports identical local_required/retained counts" + ); + } + + #[test] + fn reconciliation_deletes_refs_without_reclaiming_objects() { + let fx = fixture("no-object-reclamation"); + let orphan = fx.capture_after_writing("a.txt", "orphan\n"); + fx.pin(&orphan); + + assert_eq!(fx.object_type(&orphan.0).as_deref(), Some("tree")); + + let report = expect_reconciled(fx.reconcile()); + assert_eq!(report.deleted, 1); + assert!( + !fx.ref_exists(&fx.owned_ref(&orphan)), + "the stale ref is deleted" + ); + + assert_eq!( + fx.object_type(&orphan.0).as_deref(), + Some("tree"), + "the now-unreachable object is still resolvable — reconciliation ran \ + no git gc / git prune" + ); + } + + #[test] + fn no_checkout_identity_returns_a_distinct_skipped_outcome() { + let fx = fixture("no-identity"); + // Remove the checkout id so `read_checkout_id` returns `Ok(None)`. + let git_dir = resolve_git_dir(&fx.repo_root).expect("git dir should resolve"); + std::fs::remove_file(git_dir.join("sce").join("checkout-id")) + .expect("checkout id file should be removable"); + + let orphan = fx.capture_after_writing("a.txt", "orphan\n"); + fx.pin_for(&fx.worktree_id, &orphan); + + let outcome = fx.reconcile().expect("the skip is an Ok, not an Err"); + + assert_eq!(outcome, ReconciliationOutcome::SkippedNoCheckoutIdentity); + assert!( + fx.ref_exists(&fx.owned_ref(&orphan)), + "no pin is inventoried, validated, or deleted without a derivable identity" + ); + } + + #[test] + fn a_missing_checkout_identity_skip_touches_no_db_and_no_ref() { + let fx = fixture("no-identity-no-db"); + let git_dir = resolve_git_dir(&fx.repo_root).expect("git dir should resolve"); + std::fs::remove_file(git_dir.join("sce").join("checkout-id")) + .expect("checkout id file should be removable"); + + let orphan = fx.capture_after_writing("a.txt", "orphan\n"); + fx.pin_for(&fx.worktree_id, &orphan); + let owned = fx.owned_ref(&orphan); + let before = fx.ref_representation(&owned); + assert!( + !before.is_empty(), + "the pre-seeded pin ref must exist before the skip" + ); + + let outcome = reconcile_worktree(&fx.repo_root, || { + panic!("open_db must not be invoked on the missing-checkout-identity skip path") + }) + .expect("the skip is an Ok, not an Err"); + + assert_eq!(outcome, ReconciliationOutcome::SkippedNoCheckoutIdentity); + assert_eq!( + fx.ref_representation(&owned), + before, + "the skip touches no ref: name, target SHA, object type, and direct/symbolic \ + shape are all structurally unchanged" + ); + } +} diff --git a/cli/src/services/mutation_trace/runtime/tests.rs b/cli/src/services/mutation_trace/runtime/tests.rs index a0a2f0d4..ad8c9f87 100644 --- a/cli/src/services/mutation_trace/runtime/tests.rs +++ b/cli/src/services/mutation_trace/runtime/tests.rs @@ -1,40 +1,28 @@ use std::path::{Path, PathBuf}; use std::process::Command; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Barrier}; +use std::sync::{mpsc, Arc, Barrier}; use std::thread; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::Duration; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; use crate::services::agent_trace_storage::{ resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, }; use crate::services::checkout::{read_checkout_id, resolve_git_dir}; -use crate::services::mutation_trace::store::MutationTraceStore; +use crate::services::mutation_trace::store::{encode_revision, MutationTraceStore}; use crate::services::mutation_trace::types::{ ActorKind, EventId, FailureKind, ScopeId, ScopeStatus, }; -use super::coordinator::{coordinate, CoordinateError, RuntimeBoundary}; +use super::coordinator::{coordinate, coordinate_inner, CoordinateError, RuntimeBoundary}; use super::external_taint::ExternalTaintMarker; use super::git_snapshot::GitSnapshotService; -use super::worktree_lock::WorktreeLock; - -static NEXT_ID: AtomicU64 = AtomicU64::new(0); - -fn unique_path(label: &str) -> PathBuf { - let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after the Unix epoch") - .as_nanos(); - std::env::temp_dir().join(format!( - "sce-mutation-trace-runtime-{label}-{}-{nonce}-{id}", - std::process::id() - )) -} +use super::ref_reconciliation::{ + reconcile_worktree, reconcile_worktree_inner, ReconcileError, ReconciliationOutcome, +}; +use super::worktree_lock::{acquire_inner, WorktreeLock}; -fn run_git(dir: &Path, args: &[&str]) { +fn run_git(dir: &Path, args: &[&str]) -> String { let output = Command::new("git") .args(args) .current_dir(dir) @@ -45,6 +33,7 @@ fn run_git(dir: &Path, args: &[&str]) { "git {args:?} failed: {}", String::from_utf8_lossy(&output.stderr) ); + String::from_utf8(output.stdout).expect("git output should be valid UTF-8") } fn init_repo(root: &Path) { @@ -55,47 +44,147 @@ fn init_repo(root: &Path) { run_git(root, &["commit", "--allow-empty", "--quiet", "-m", "init"]); } -fn cleanup(path: &Path) { - let _ = std::fs::remove_dir_all(path); +struct TestRepo { + _temp_dir: tempfile::TempDir, + repo_root: PathBuf, + db_path: PathBuf, +} + +impl TestRepo { + fn new(label: &str) -> Self { + let temp_dir = tempfile::Builder::new() + .prefix(&format!("sce-mutation-trace-runtime-{label}-")) + .tempdir() + .expect("test temp directory should be created"); + let repo_root = temp_dir.path().join("repo"); + init_repo(&repo_root); + let db_path = temp_dir.path().join("agent-trace.db"); + RepositoryAgentTraceDb::new_at(&db_path) + .expect("the repository DB should open with schema"); + Self { + _temp_dir: temp_dir, + repo_root, + db_path, + } + } + + fn open_db(&self) -> anyhow::Result { + RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&self.db_path) + } + + fn db(&self) -> RepositoryAgentTraceDb { + self.open_db() + .expect("reopening the DB for assertions should succeed") + } +} + +struct LinkedTestRepo { + _temp_dir: tempfile::TempDir, + main_root: PathBuf, + linked_root: PathBuf, + db_path: PathBuf, +} + +impl LinkedTestRepo { + fn new(label: &str) -> Self { + let temp_dir = tempfile::Builder::new() + .prefix(&format!("sce-mutation-trace-runtime-{label}-")) + .tempdir() + .expect("test temp directory should be created"); + let main_root = temp_dir.path().join("main"); + init_repo(&main_root); + let linked_root = temp_dir.path().join("linked"); + run_git( + &main_root, + &[ + "worktree", + "add", + "--quiet", + linked_root.to_str().expect("worktree path should be UTF-8"), + ], + ); + let db_path = temp_dir.path().join("agent-trace.db"); + RepositoryAgentTraceDb::new_at(&db_path) + .expect("the shared repository DB should open with schema"); + Self { + _temp_dir: temp_dir, + main_root, + linked_root, + db_path, + } + } + + fn open_db(&self) -> anyhow::Result { + RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&self.db_path) + } + + fn db(&self) -> RepositoryAgentTraceDb { + self.open_db() + .expect("reopening the shared DB for assertions should succeed") + } +} + +fn seed_event( + db: &RepositoryAgentTraceDb, + worktree_id: &str, + revision: u64, + before_tree: &str, + after_tree: &str, +) { + db.execute( + "INSERT INTO mutation_trace_events + (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, + attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, + boundary_event_id) + VALUES (?1, ?2, ?3, ?4, 0, 'healthy', 'ineligible_unscoped', NULL, 'flush', NULL, NULL)", + ( + worktree_id, + encode_revision(revision).as_slice(), + before_tree, + after_tree, + ), + ) + .expect("event row insert should succeed"); +} + +fn ref_exists(dir: &Path, ref_name: &str) -> bool { + Command::new("git") + .args(["show-ref", "--verify", "--quiet", ref_name]) + .current_dir(dir) + .status() + .expect("git show-ref should spawn") + .success() +} + +fn row_count(db: &RepositoryAgentTraceDb, table: &str) -> i64 { + db.query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { + row.get::(0).map_err(Into::into) + }) + .expect("count query should succeed") + .into_iter() + .next() + .expect("count row should exist") } #[test] fn linked_worktrees_have_independent_locks_and_worktree_ids() { - let main_root = unique_path("linked-main"); - init_repo(&main_root); - let linked_root = unique_path("linked-secondary"); - run_git( - &main_root, - &[ - "worktree", - "add", - "--quiet", - linked_root.to_str().expect("worktree path should be UTF-8"), - ], - ); - - let db_path = main_root.join("agent-trace.db"); - let db_main = RepositoryAgentTraceDb::new_at(&db_path).expect("main repository DB should open"); + let repo = LinkedTestRepo::new("linked-ids"); - let main_git_dir = resolve_git_dir(&main_root).expect("main git dir should resolve"); - let linked_git_dir = resolve_git_dir(&linked_root).expect("linked git dir should resolve"); + let main_git_dir = resolve_git_dir(&repo.main_root).expect("main git dir should resolve"); + let linked_git_dir = resolve_git_dir(&repo.linked_root).expect("linked git dir should resolve"); assert_ne!( main_git_dir, linked_git_dir, "a linked worktree must resolve its own worktree-specific git dir, giving it a distinct lock and identity path" ); - let main_outcome = coordinate(&main_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }) - .expect("first observation on the main worktree should succeed"); + let main_outcome = coordinate(&repo.main_root, &RuntimeBoundary::Flush, || repo.open_db()) + .expect("first observation on the main worktree should succeed"); let held = WorktreeLock::acquire(&main_git_dir, Duration::from_secs(5)) .expect("the main worktree's runtime lock should be acquirable"); - let linked_outcome = coordinate(&linked_root, &RuntimeBoundary::Flush, || { - // A second handle to the same caller-supplied repository-scoped DB path, - // opened through the provider while the main worktree's lock is held. - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) + let linked_outcome = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, || { + repo.open_db() }) .expect( "coordinate() on the linked worktree must acquire its own distinct runtime lock while the main worktree's lock is still held", @@ -108,6 +197,7 @@ fn linked_worktrees_have_independent_locks_and_worktree_ids() { "each linked worktree must derive a distinct WorktreeId from its own checkout identity" ); + let db_main = repo.db(); let store = MutationTraceStore::new(&db_main); assert!( store @@ -124,35 +214,30 @@ fn linked_worktrees_have_independent_locks_and_worktree_ids() { "the linked worktree's coordinator should have persisted its distinct row into the same caller-supplied DB" ); - let linked_snapshot = GitSnapshotService::new(&linked_root) + let linked_snapshot = GitSnapshotService::new(&repo.linked_root) .expect("a snapshot service should construct for the linked worktree"); linked_snapshot .diff_trees(&main_outcome.observed_tree, &main_outcome.observed_tree) .expect("a tree pinned by the main worktree's coordinator must resolve through the linked worktree's git dir"); - - cleanup(&linked_root); - cleanup(&main_root); } #[test] fn agent_trace_storage_and_coordinator_observe_the_same_checkout_id() { - let repo_root = unique_path("cross-caller-repo"); - init_repo(&repo_root); + let repo = TestRepo::new("cross-caller"); run_git( - &repo_root, + &repo.repo_root, &["remote", "add", "origin", "git@github.com:acme/widgets.git"], ); - let state_root = unique_path("cross-caller-state"); - std::fs::create_dir_all(&state_root).expect("state root should be created"); - - let coordinator_db_path = repo_root.join("coordinator.db"); - RepositoryAgentTraceDb::new_at(&coordinator_db_path) - .expect("the coordinator's repository DB should open with schema"); + let state_dir = tempfile::Builder::new() + .prefix("sce-mutation-trace-runtime-cross-caller-state-") + .tempdir() + .expect("state root temp directory should be created"); + let state_root = state_dir.path().to_path_buf(); let barrier = Arc::new(Barrier::new(2)); let storage_thread = { - let repo_root = repo_root.clone(); + let repo_root = repo.repo_root.clone(); let state_root = state_root.clone(); let barrier = Arc::clone(&barrier); thread::spawn(move || { @@ -169,10 +254,8 @@ fn agent_trace_storage_and_coordinator_observe_the_same_checkout_id() { }; barrier.wait(); - let outcome = coordinate(&repo_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&coordinator_db_path) - }) - .expect("the coordinator's first observation should succeed"); + let outcome = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, || repo.open_db()) + .expect("the coordinator's first observation should succeed"); let storage_checkout_id = storage_thread .join() @@ -183,32 +266,26 @@ fn agent_trace_storage_and_coordinator_observe_the_same_checkout_id() { "the coordinator and agent_trace_storage must converge on one checkout identity for the same physical checkout" ); - let on_disk = read_checkout_id(&resolve_git_dir(&repo_root).expect("git dir should resolve")) - .expect("reading the checkout-id file should succeed") - .expect("a checkout id must have been persisted"); + let on_disk = + read_checkout_id(&resolve_git_dir(&repo.repo_root).expect("git dir should resolve")) + .expect("reading the checkout-id file should succeed") + .expect("a checkout id must have been persisted"); assert_eq!( on_disk, storage_checkout_id, "the on-disk checkout-id file must contain the converged identity" ); - - cleanup(&repo_root); - cleanup(&state_root); } #[test] fn a_snapshot_failure_then_recovery_cycle_runs_through_the_public_api() { - let repo_root = unique_path("failure-recovery-repo"); - init_repo(&repo_root); - let db_path = repo_root.join("agent-trace.db"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let repo = TestRepo::new("failure-recovery"); + let ok_db = || repo.open_db(); - let baseline = coordinate(&repo_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }) - .expect("the baseline observation should materialize the worktree"); + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) + .expect("the baseline observation should materialize the worktree"); let worktree_id = baseline.worktree_id.clone(); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); + let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); let tmp_index_dir = git_dir.join("sce").join("tmp"); let _ = std::fs::remove_dir_all(&tmp_index_dir); std::fs::write(&tmp_index_dir, b"not a directory").expect( @@ -217,13 +294,13 @@ fn a_snapshot_failure_then_recovery_cycle_runs_through_the_public_api() { let scope = ScopeId("scope-recovery".to_string()); let failure = coordinate( - &repo_root, + &repo.repo_root, &RuntimeBoundary::Start { scope: scope.clone(), event: EventId("evt-during-failure".to_string()), actor_kind: ActorKind::ClaudeCode, }, - || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path), + ok_db, ) .expect_err("a Git snapshot failure against a materialized worktree should be reported"); match failure { @@ -237,6 +314,7 @@ fn a_snapshot_failure_then_recovery_cycle_runs_through_the_public_api() { } { + let db = repo.db(); let store = MutationTraceStore::new(&db); let projection = store .load_worktree(&worktree_id, None, None) @@ -251,15 +329,14 @@ fn a_snapshot_failure_then_recovery_cycle_runs_through_the_public_api() { std::fs::remove_file(&tmp_index_dir) .expect("removing the planted file should let the snapshot service recreate its temp dir"); - let recovered = coordinate(&repo_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }) - .expect("the coordinator should recover from the taint and process the boundary"); + let recovered = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) + .expect("the coordinator should recover from the taint and process the boundary"); assert_eq!( recovered.worktree_id, worktree_id, "recovery must operate on the same worktree identity" ); + let db = repo.db(); let store = MutationTraceStore::new(&db); let projection = store .load_worktree(&worktree_id, None, None) @@ -269,24 +346,16 @@ fn a_snapshot_failure_then_recovery_cycle_runs_through_the_public_api() { !projection.worktree_state.tainted, "taint recovery must clear the tainted flag before the triggering boundary is processed" ); - - cleanup(&repo_root); } #[test] fn a_successful_coordinate_through_the_public_api_leaves_no_external_taint_marker() { - let repo_root = unique_path("public-success-no-marker"); - init_repo(&repo_root); - // The DB lives outside the worktree so it never perturbs the observed tree. - let db_path = unique_path("public-success-no-marker-db").join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path).expect("the repository DB should open with schema"); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); + let repo = TestRepo::new("public-success-no-marker"); + let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); let marker = ExternalTaintMarker::new(&git_dir); - let outcome = coordinate(&repo_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }) - .expect("a first observation through the public entrypoint should succeed"); + let outcome = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, || repo.open_db()) + .expect("a first observation through the public entrypoint should succeed"); assert_eq!( outcome.revision, 0, "a first-observation flush should not advance the revision" @@ -297,35 +366,23 @@ fn a_successful_coordinate_through_the_public_api_leaves_no_external_taint_marke .expect("marker existence should resolve after a successful coordinate()"), "a successful coordinate() must clear the external-taint marker it armed" ); - - cleanup( - db_path - .parent() - .expect("the DB path has a parent directory"), - ); - cleanup(&repo_root); } #[test] #[allow(clippy::too_many_lines)] fn a_db_open_failure_after_arming_leaves_the_marker_and_the_next_invocation_rebaselines_without_evidence( ) { - let repo_root = unique_path("public-db-open-failure-gap"); - init_repo(&repo_root); - // The DB lives outside the worktree so it never perturbs the observed tree. - let db_path = unique_path("public-db-open-failure-gap-db").join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path).expect("the repository DB should open with schema"); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); + let repo = TestRepo::new("public-db-open-failure-gap"); + let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); let marker = ExternalTaintMarker::new(&git_dir); - let ok_db = || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path); + let ok_db = || repo.open_db(); - // A trusted baseline at cursor A, then one exclusive AI edit A -> B. - let baseline = coordinate(&repo_root, &RuntimeBoundary::Flush, ok_db) + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) .expect("the baseline observation should materialize the worktree"); let worktree_id = baseline.worktree_id.clone(); let scope = ScopeId("scope-across-the-gap".to_string()); coordinate( - &repo_root, + &repo.repo_root, &RuntimeBoundary::Start { scope: scope.clone(), event: EventId("evt-start".to_string()), @@ -334,9 +391,9 @@ fn a_db_open_failure_after_arming_leaves_the_marker_and_the_next_invocation_reba ok_db, ) .expect("starting the scope should succeed"); - std::fs::write(repo_root.join("work.txt"), b"v1").expect("the A -> B edit should write"); + std::fs::write(repo.repo_root.join("work.txt"), b"v1").expect("the A -> B edit should write"); let advanced = coordinate( - &repo_root, + &repo.repo_root, &RuntimeBoundary::Advance { scope: scope.clone(), event: EventId("evt-advance".to_string()), @@ -351,10 +408,8 @@ fn a_db_open_failure_after_arming_leaves_the_marker_and_the_next_invocation_reba "the exclusive A -> B edit must land one trustworthy event before the gap" ); - // A boundary invocation whose DB provider fails after the marker is armed, - // then the working tree keeps moving during the lost interval: B -> C. let db_failure = coordinate( - &repo_root, + &repo.repo_root, &RuntimeBoundary::Flush, || -> anyhow::Result { Err(anyhow::anyhow!("simulated Agent Trace DB open failure")) @@ -369,10 +424,9 @@ fn a_db_open_failure_after_arming_leaves_the_marker_and_the_next_invocation_reba marker.exists().expect("marker existence should resolve"), "the armed marker must survive an invocation whose DB provider returned Err" ); - std::fs::write(repo_root.join("work.txt"), b"v2-during-the-gap").expect("the B -> C edit"); + std::fs::write(repo.repo_root.join("work.txt"), b"v2-during-the-gap").expect("the B -> C edit"); - // The next successful invocation rebaselines to C with no evidence for the gap. - let recovered = coordinate(&repo_root, &RuntimeBoundary::Flush, ok_db) + let recovered = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) .expect("the follow-up invocation with a working provider should recover, then process its boundary"); let tree_c = recovered.observed_tree.clone(); assert_ne!( @@ -387,8 +441,7 @@ fn a_db_open_failure_after_arming_leaves_the_marker_and_the_next_invocation_reba !marker.exists().expect("marker existence should resolve"), "the recovering invocation must clear the marker on success" ); - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - .expect("reopening the DB for assertions should succeed"); + let db = repo.db(); let store = MutationTraceStore::new(&db); let projection = store .load_worktree(&worktree_id, Some(&scope), None) @@ -415,34 +468,22 @@ fn a_db_open_failure_after_arming_leaves_the_marker_and_the_next_invocation_reba ); } } - - cleanup( - db_path - .parent() - .expect("the DB path has a parent directory"), - ); - cleanup(&repo_root); } #[test] fn a_stale_marker_rebaselines_to_the_current_tree_abandons_scopes_then_processes_the_boundary() { - let repo_root = unique_path("public-stale-marker-rebaseline"); - init_repo(&repo_root); - // The DB lives outside the worktree so it never perturbs the observed tree. - let db_path = unique_path("public-stale-marker-rebaseline-db").join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path).expect("the repository DB should open with schema"); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); + let repo = TestRepo::new("public-stale-marker-rebaseline"); + let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); let marker = ExternalTaintMarker::new(&git_dir); - let ok_db = || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path); + let ok_db = || repo.open_db(); - // Trusted cursor A plus an active scope S. - let baseline = coordinate(&repo_root, &RuntimeBoundary::Flush, ok_db) + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) .expect("the baseline observation should materialize the worktree"); let worktree_id = baseline.worktree_id.clone(); let tree_a = baseline.observed_tree.clone(); let scope = ScopeId("scope-stranded".to_string()); coordinate( - &repo_root, + &repo.repo_root, &RuntimeBoundary::Start { scope: scope.clone(), event: EventId("evt-start".to_string()), @@ -452,21 +493,18 @@ fn a_stale_marker_rebaselines_to_the_current_tree_abandons_scopes_then_processes ) .expect("starting scope S should succeed"); - // A crashed invocation: the marker was armed and never cleared. marker .persist() .expect("simulating a crashed invocation that armed but never cleared the marker"); - // The working tree moves on to C while the process was gone. std::fs::write( - repo_root.join("stranded.txt"), + repo.repo_root.join("stranded.txt"), b"edited-while-the-process-was-gone", ) .expect("the A -> C edit should write"); - // The next invocation inherits the stale marker. let recovered = coordinate( - &repo_root, + &repo.repo_root, &RuntimeBoundary::Advance { scope: scope.clone(), event: EventId("evt-advance".to_string()), @@ -489,8 +527,7 @@ fn a_stale_marker_rebaselines_to_the_current_tree_abandons_scopes_then_processes "a successful recovery must clear the inherited marker" ); - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - .expect("reopening the DB for assertions should succeed"); + let db = repo.db(); let store = MutationTraceStore::new(&db); let projection = store .load_worktree(&worktree_id, Some(&scope), None) @@ -507,43 +544,29 @@ fn a_stale_marker_rebaselines_to_the_current_tree_abandons_scopes_then_processes "the scope that was live across the gap must be abandoned during recovery" ); - // The triggering boundary was processed inside the inheriting invocation: - // a plain follow-up flush finds nothing left to recover or advance. - let stable = coordinate(&repo_root, &RuntimeBoundary::Flush, ok_db) + let stable = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) .expect("a plain flush after recovery should succeed"); assert_eq!( stable.revision, recovered.revision, "recovery and the triggering boundary already completed in the inheriting invocation" ); assert!(stable.mutation_event.is_none()); - - cleanup( - db_path - .parent() - .expect("the DB path has a parent directory"), - ); - cleanup(&repo_root); } #[test] fn a_first_ever_failed_invocation_that_never_materialized_a_worktree_row_creates_no_evidence() { - let repo_root = unique_path("public-first-ever-failure"); - init_repo(&repo_root); - // The DB lives outside the worktree so it never perturbs the observed tree. - let db_path = unique_path("public-first-ever-failure-db").join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path).expect("the repository DB should open with schema"); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); + let repo = TestRepo::new("public-first-ever-failure"); + let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); let marker = ExternalTaintMarker::new(&git_dir); std::fs::write( - repo_root.join("pre-existing.txt"), + repo.repo_root.join("pre-existing.txt"), b"content before any observation", ) .expect("a pre-existing edit should write"); - // The very first invocation fails opening the DB, after arming the marker. let first = coordinate( - &repo_root, + &repo.repo_root, &RuntimeBoundary::Flush, || -> anyhow::Result { Err(anyhow::anyhow!( @@ -561,19 +584,14 @@ fn a_first_ever_failed_invocation_that_never_materialized_a_worktree_row_creates "the first-ever failed invocation still leaves an armed marker" ); - // More edits during the still-unobserved interval. std::fs::write( - repo_root.join("during-the-gap.txt"), + repo.repo_root.join("during-the-gap.txt"), b"more unknown-interval content", ) .expect("another unobserved edit should write"); - // The first *successful* invocation establishes a baseline with no evidence - // for the unknown interval. - let established = coordinate(&repo_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }) - .expect("the first successful invocation establishes the baseline"); + let established = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, || repo.open_db()) + .expect("the first successful invocation establishes the baseline"); assert!( established.mutation_event.is_none(), "a worktree with no prior durable row cannot produce evidence for the unknown interval" @@ -583,8 +601,7 @@ fn a_first_ever_failed_invocation_that_never_materialized_a_worktree_row_creates "the successful baseline must clear the inherited marker" ); - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - .expect("reopening the DB for assertions should succeed"); + let db = repo.db(); let store = MutationTraceStore::new(&db); let projection = store .load_worktree(&established.worktree_id, None, None) @@ -605,75 +622,46 @@ fn a_first_ever_failed_invocation_that_never_materialized_a_worktree_row_creates "no MutationEvent may exist for a worktree whose history began with an unobserved interval" ); } - - cleanup( - db_path - .parent() - .expect("the DB path has a parent directory"), - ); - cleanup(&repo_root); } #[test] #[allow(clippy::too_many_lines)] fn linked_worktrees_keep_independent_external_taint_markers_over_a_shared_db() { - let main_root = unique_path("public-taint-linked-main"); - init_repo(&main_root); - let linked_root = unique_path("public-taint-linked-secondary"); - run_git( - &main_root, - &[ - "worktree", - "add", - "--quiet", - linked_root.to_str().expect("worktree path should be UTF-8"), - ], - ); - - // One shared repository DB, outside either worktree so it never perturbs a - // captured tree. - let db_path = unique_path("public-taint-linked-db").join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path).expect("the shared repository DB should open"); + let repo = LinkedTestRepo::new("public-taint-linked"); - let main_git_dir = resolve_git_dir(&main_root).expect("main git dir should resolve"); - let linked_git_dir = resolve_git_dir(&linked_root).expect("linked git dir should resolve"); + let main_git_dir = resolve_git_dir(&repo.main_root).expect("main git dir should resolve"); + let linked_git_dir = resolve_git_dir(&repo.linked_root).expect("linked git dir should resolve"); let main_marker = ExternalTaintMarker::new(&main_git_dir); let linked_marker = ExternalTaintMarker::new(&linked_git_dir); - // Baseline both worktrees against the one shared DB. - let main_baseline = coordinate(&main_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }) - .expect("the main worktree baseline should succeed"); - let linked_baseline = coordinate(&linked_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) + let main_baseline = coordinate(&repo.main_root, &RuntimeBoundary::Flush, || repo.open_db()) + .expect("the main worktree baseline should succeed"); + let linked_baseline = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, || { + repo.open_db() }) .expect("the linked worktree baseline should succeed"); assert_ne!(main_baseline.worktree_id, linked_baseline.worktree_id); - // Strand a live scope in the linked worktree behind an armed marker. let linked_scope = ScopeId("scope-linked".to_string()); coordinate( - &linked_root, + &repo.linked_root, &RuntimeBoundary::Start { scope: linked_scope.clone(), event: EventId("evt-linked-start".to_string()), actor_kind: ActorKind::ClaudeCode, }, - || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path), + || repo.open_db(), ) .expect("starting the linked worktree's scope should succeed"); linked_marker .persist() .expect("arming the linked worktree's marker should succeed"); - // A boundary in the MAIN worktree must not observe the linked worktree's marker. - std::fs::write(main_root.join("main-work.txt"), b"v1") + std::fs::write(repo.main_root.join("main-work.txt"), b"v1") .expect("a main-worktree edit should write"); - coordinate(&main_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }) - .expect("the main worktree flush must succeed without inheriting the linked worktree's marker"); + coordinate(&repo.main_root, &RuntimeBoundary::Flush, || repo.open_db()).expect( + "the main worktree flush must succeed without inheriting the linked worktree's marker", + ); assert!( !main_marker .exists() @@ -687,8 +675,7 @@ fn linked_worktrees_keep_independent_external_taint_markers_over_a_shared_db() { "the main worktree's invocation must not touch the linked worktree's independent marker" ); - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - .expect("reopening the shared DB for assertions should succeed"); + let db = repo.db(); let store = MutationTraceStore::new(&db); let linked_mid = store .load_worktree(&linked_baseline.worktree_id, Some(&linked_scope), None) @@ -700,11 +687,10 @@ fn linked_worktrees_keep_independent_external_taint_markers_over_a_shared_db() { "a marker in the linked worktree must not trigger recovery of the linked worktree from the main worktree's invocation" ); - // The linked worktree's own next invocation inherits and recovers. - std::fs::write(linked_root.join("linked-work.txt"), b"v1") + std::fs::write(repo.linked_root.join("linked-work.txt"), b"v1") .expect("a linked-worktree edit should write"); - let linked_recovered = coordinate(&linked_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) + let linked_recovered = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, || { + repo.open_db() }) .expect("the linked worktree's own invocation recovers from its inherited marker"); assert!( @@ -727,33 +713,20 @@ fn linked_worktrees_keep_independent_external_taint_markers_over_a_shared_db() { Some(ScopeStatus::Abandoned), "the linked worktree's inherited-taint recovery abandons its own live scope" ); - - cleanup( - db_path - .parent() - .expect("the DB path has a parent directory"), - ); - cleanup(&linked_root); - cleanup(&main_root); } #[test] fn a_snapshot_failure_arms_the_marker_and_the_next_invocation_recovers_once() { - let repo_root = unique_path("public-snapshot-failure-marker-recovery"); - init_repo(&repo_root); - // The DB lives outside the worktree so it never perturbs the observed tree. - let db_path = unique_path("public-snapshot-failure-marker-recovery-db").join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path).expect("the repository DB should open with schema"); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); + let repo = TestRepo::new("public-snapshot-failure-marker-recovery"); + let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); let marker = ExternalTaintMarker::new(&git_dir); - let ok_db = || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path); + let ok_db = || repo.open_db(); - let baseline = coordinate(&repo_root, &RuntimeBoundary::Flush, ok_db) + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) .expect("the baseline observation should materialize the worktree"); let worktree_id = baseline.worktree_id.clone(); assert!(!marker.exists().expect("marker existence should resolve")); - // Plant a file where the snapshot service needs its temp-index directory. let tmp_index_dir = git_dir.join("sce").join("tmp"); let _ = std::fs::remove_dir_all(&tmp_index_dir); std::fs::write(&tmp_index_dir, b"not a directory") @@ -761,7 +734,7 @@ fn a_snapshot_failure_arms_the_marker_and_the_next_invocation_recovers_once() { let scope = ScopeId("scope-during-the-failure".to_string()); let failure = coordinate( - &repo_root, + &repo.repo_root, &RuntimeBoundary::Start { scope: scope.clone(), event: EventId("evt-during-failure".to_string()), @@ -786,8 +759,7 @@ fn a_snapshot_failure_arms_the_marker_and_the_next_invocation_recovers_once() { ); { - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - .expect("reopening the DB for assertions should succeed"); + let db = repo.db(); let store = MutationTraceStore::new(&db); let projection = store .load_worktree(&worktree_id, None, None) @@ -802,10 +774,10 @@ fn a_snapshot_failure_arms_the_marker_and_the_next_invocation_recovers_once() { std::fs::remove_file(&tmp_index_dir) .expect("removing the planted file should let the snapshot service recreate its temp dir"); - std::fs::write(repo_root.join("work.txt"), b"v1") + std::fs::write(repo.repo_root.join("work.txt"), b"v1") .expect("an edit before recovery should write"); - let recovered = coordinate(&repo_root, &RuntimeBoundary::Flush, ok_db) + let recovered = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) .expect("the coordinator should recover from the taint and process the boundary"); assert!( recovered.mutation_event.is_none(), @@ -816,8 +788,7 @@ fn a_snapshot_failure_arms_the_marker_and_the_next_invocation_recovers_once() { "a successful recovery clears the marker" ); - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - .expect("reopening the DB for assertions should succeed"); + let db = repo.db(); let store = MutationTraceStore::new(&db); let projection = store .load_worktree(&worktree_id, None, None) @@ -833,44 +804,31 @@ fn a_snapshot_failure_arms_the_marker_and_the_next_invocation_recovers_once() { "recovery rebaselines the cursor to the recovering invocation's own observed tree" ); - // A plain follow-up flush proves the recovery already fully completed once: - // there is nothing left to recover. - let stable = coordinate(&repo_root, &RuntimeBoundary::Flush, ok_db) + let stable = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) .expect("a plain flush after recovery should succeed"); assert_eq!( stable.revision, recovered.revision, "the single conservative recovery already completed; a follow-up flush is a no-op" ); assert!(stable.mutation_event.is_none()); - - cleanup( - db_path - .parent() - .expect("the DB path has a parent directory"), - ); - cleanup(&repo_root); } #[test] #[allow(clippy::too_many_lines)] fn a_marker_clear_failure_after_a_durable_boundary_keeps_the_marker_for_a_later_recovery() { - let repo_root = unique_path("public-marker-clear-failure"); - init_repo(&repo_root); - // The DB lives outside the worktree so it never perturbs the observed tree. - let db_path = unique_path("public-marker-clear-failure-db").join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path).expect("the repository DB should open with schema"); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); + let repo = TestRepo::new("public-marker-clear-failure"); + let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); let marker = ExternalTaintMarker::new(&git_dir); let marker_path = git_dir.join("sce").join("mutation-cursor-tainted"); - let ok_db = || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path); + let ok_db = || repo.open_db(); - let baseline = coordinate(&repo_root, &RuntimeBoundary::Flush, ok_db) + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) .expect("the baseline observation should materialize the worktree"); let worktree_id = baseline.worktree_id.clone(); let scope = ScopeId("scope-attributable".to_string()); coordinate( - &repo_root, + &repo.repo_root, &RuntimeBoundary::Start { scope: scope.clone(), event: EventId("evt-start".to_string()), @@ -879,13 +837,13 @@ fn a_marker_clear_failure_after_a_durable_boundary_keeps_the_marker_for_a_later_ ok_db, ) .expect("starting the scope should succeed"); - std::fs::write(repo_root.join("work.txt"), b"v1") + std::fs::write(repo.repo_root.join("work.txt"), b"v1") .expect("an exclusive edit before the boundary should write"); let clear_marker_path = marker_path.clone(); - let clear_db_path = db_path.clone(); + let clear_db_path = repo.db_path.clone(); let error = coordinate( - &repo_root, + &repo.repo_root, &RuntimeBoundary::Advance { scope: scope.clone(), event: EventId("evt-advance".to_string()), @@ -911,8 +869,7 @@ fn a_marker_clear_failure_after_a_durable_boundary_keeps_the_marker_for_a_later_ ); let (durable_revision, durable_cursor) = { - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - .expect("reopening the DB for assertions should succeed"); + let db = repo.db(); let store = MutationTraceStore::new(&db); let projection = store .load_worktree(&worktree_id, Some(&scope), None) @@ -941,9 +898,9 @@ fn a_marker_clear_failure_after_a_durable_boundary_keeps_the_marker_for_a_later_ marker .persist() .expect("re-arming a plain marker file should succeed"); - std::fs::write(repo_root.join("work.txt"), b"v2").expect("a later edit should write"); + std::fs::write(repo.repo_root.join("work.txt"), b"v2").expect("a later edit should write"); - let recovered = coordinate(&repo_root, &RuntimeBoundary::Flush, ok_db) + let recovered = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) .expect("the later invocation recovers from the still-armed marker"); assert!( recovered.mutation_event.is_none(), @@ -957,11 +914,938 @@ fn a_marker_clear_failure_after_a_durable_boundary_keeps_the_marker_for_a_later_ recovered.observed_tree, durable_cursor, "the later recovery rebaselines to the newer tree" ); +} + +#[test] +fn reconciliation_blocks_on_the_worktree_lock_and_retains_a_pin_that_becomes_durable_under_it() { + let repo = TestRepo::new("reconcile-blocks-on-worktree-lock"); + let ok_db = || repo.open_db(); - cleanup( - db_path - .parent() - .expect("the DB path has a parent directory"), + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) + .expect("the baseline observation should materialize the worktree"); + let worktree_id = baseline.worktree_id.clone(); + let baseline_tree = baseline.observed_tree.clone(); + + let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); + let held = acquire_inner(&git_dir, Duration::from_secs(5), || {}) + .expect("the test should hold a real WorktreeLock before the worker runs"); + + let (contention_tx, contention_rx) = mpsc::channel(); + let (result_tx, result_rx) = mpsc::channel(); + let repo_root_clone = repo.repo_root.clone(); + let db_path_clone = repo.db_path.clone(); + let worker = thread::spawn(move || { + let outcome = reconcile_worktree_inner( + &repo_root_clone, + || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path_clone), + move || { + contention_tx + .send(()) + .expect("contention signal channel should still be open"); + }, + ); + result_tx + .send(()) + .expect("result signal channel should still be open"); + outcome + }); + + contention_rx.recv_timeout(Duration::from_secs(5)).expect( + "reconcile_worktree_inner should reach the WorktreeLock try_lock loop and \ + observe contention while this test still holds the lock", + ); + assert!( + result_rx.recv_timeout(Duration::from_millis(300)).is_err(), + "the reconciliation pass must not complete while the worktree lock is still held" + ); + + let snapshot = GitSnapshotService::new(&repo.repo_root) + .expect("a snapshot service should construct for the worktree"); + std::fs::write(repo.repo_root.join("under-lock.txt"), b"under the lock") + .expect("an edit under the lock should write"); + let x = snapshot.capture_tree().expect("capturing X should succeed"); + snapshot + .pin_tree(&worktree_id, &x) + .expect("pinning X should succeed"); + seed_event( + &ok_db().expect("reopening the DB to seed the event should succeed"), + &worktree_id.0, + 1, + &baseline_tree.0, + &x.0, + ); + + drop(held); + + result_rx + .recv_timeout(Duration::from_secs(5)) + .expect("the same reconciliation pass should complete once the lock is released"); + let outcome = worker + .join() + .expect("the reconciliation worker thread should not panic") + .expect( + "reconciliation should succeed once it can acquire the worktree lock after release", + ); + let report = match outcome { + ReconciliationOutcome::Reconciled(report) => report, + ReconciliationOutcome::SkippedNoCheckoutIdentity => { + panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") + } + }; + + assert_eq!( + report.deleted, 0, + "reconciliation must retain X: it became a durable root under the very lock it was waiting on" + ); + assert_eq!( + report.local_required, 2, + "the worktree's durable roots are exactly the baseline cursor tree and X" + ); + run_git( + &repo.repo_root, + &[ + "show-ref", + "--verify", + "--quiet", + &format!("refs/sce/mutation-cursor/{}/{}", worktree_id.0, x.0), + ], + ); +} + +#[test] +#[allow(clippy::too_many_lines)] +fn reconciliation_blocks_until_a_real_coordinate_cas_commits_the_pinned_tree() { + let repo = TestRepo::new("reconcile-blocks-until-real-cas"); + let ok_db = || repo.open_db(); + + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) + .expect("the baseline observation should materialize the worktree"); + let worktree_id = baseline.worktree_id.clone(); + let baseline_tree = baseline.observed_tree.clone(); + + std::fs::write( + repo.repo_root.join("under-real-cas.txt"), + b"observed before the real CAS", + ) + .expect("an edit before the coordinated boundary should write"); + + let (pinned_tx, pinned_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel::<()>(); + let (coordinate_done_tx, coordinate_done_rx) = mpsc::channel(); + let coord_repo_root = repo.repo_root.clone(); + let coord_db_path = repo.db_path.clone(); + let coordinator = thread::spawn(move || { + let mut paused = false; + let outcome = coordinate_inner( + &coord_repo_root, + &RuntimeBoundary::Flush, + || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&coord_db_path), + || {}, + move |_attempt| { + if !paused { + paused = true; + pinned_tx + .send(()) + .expect("the pin-done signal channel should still be open"); + release_rx + .recv() + .expect("the release channel should deliver before the real CAS"); + } + }, + |_attempt| Ok(()), + ); + coordinate_done_tx + .send(()) + .expect("the coordinate-done signal channel should still be open"); + outcome + }); + + pinned_rx.recv_timeout(Duration::from_secs(5)).expect( + "coordinate_inner should pin X, load the worktree, and pause in after_load \ + while still holding the WorktreeLock", + ); + + let (contention_tx, contention_rx) = mpsc::channel(); + let (reconcile_done_tx, reconcile_done_rx) = mpsc::channel(); + let rec_repo_root = repo.repo_root.clone(); + let rec_db_path = repo.db_path.clone(); + let reconciler = thread::spawn(move || { + let outcome = reconcile_worktree_inner( + &rec_repo_root, + || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&rec_db_path), + move || { + contention_tx + .send(()) + .expect("the contention signal channel should still be open"); + }, + ); + reconcile_done_tx + .send(()) + .expect("the reconcile-done signal channel should still be open"); + outcome + }); + + contention_rx.recv_timeout(Duration::from_secs(5)).expect( + "reconcile_worktree_inner should observe WorktreeLock contention while the \ + real coordinate() CAS is still pending", + ); + assert!( + reconcile_done_rx + .recv_timeout(Duration::from_millis(300)) + .is_err(), + "reconciliation must not complete while coordinate() still holds the lock across pin -> CAS" + ); + + release_tx + .send(()) + .expect("releasing the coordinate worker should succeed"); + + coordinate_done_rx + .recv_timeout(Duration::from_secs(5)) + .expect("the real coordinate() invocation should complete after the CAS"); + let coordinate_outcome = coordinator + .join() + .expect("the coordinate worker thread should not panic") + .expect("the real coordinate() CAS should apply the observed drift"); + let x = coordinate_outcome.observed_tree.clone(); + assert_ne!( + x, baseline_tree, + "the coordinated Flush must observe a real drift from the baseline tree" + ); + assert!( + coordinate_outcome.mutation_event.is_some(), + "the observed drift must commit exactly one durable MutationEvent through the real CAS" + ); + + reconcile_done_rx + .recv_timeout(Duration::from_secs(5)) + .expect("reconciliation should complete once the coordinator releases the lock"); + let outcome = reconciler + .join() + .expect("the reconciliation worker thread should not panic") + .expect( + "reconciliation should succeed once it can acquire the worktree lock after release", + ); + let report = match outcome { + ReconciliationOutcome::Reconciled(report) => report, + ReconciliationOutcome::SkippedNoCheckoutIdentity => { + panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") + } + }; + assert_eq!( + report.deleted, 0, + "reconciliation must retain X: it became a durable root through the real coordinate() CAS \ + under the very lock reconciliation was waiting on" + ); + assert_eq!( + report.local_required, 2, + "the worktree's durable roots are exactly the baseline cursor tree and X" + ); + + run_git( + &repo.repo_root, + &[ + "show-ref", + "--verify", + "--quiet", + &format!("refs/sce/mutation-cursor/{}/{}", worktree_id.0, x.0), + ], + ); + + let db = ok_db().expect("reopening the DB for assertions should succeed"); + let store = MutationTraceStore::new(&db); + let projection = store + .load_worktree(&worktree_id, None, None) + .expect("loading the worktree row should succeed") + .expect("the worktree row should exist"); + assert_eq!( + projection.worktree_state.cursor_tree, x, + "the real coordinator CAS advanced the durable cursor to X" + ); + let event = store + .load_mutation_event(&worktree_id, projection.worktree_state.revision) + .expect("loading the committed mutation event should succeed") + .expect("the observed drift must have committed one durable event"); + assert_eq!(event.before_tree, baseline_tree); + assert_eq!(event.after_tree, x); +} + +#[test] +fn a_pin_with_no_durable_root_is_reclaimed_by_a_later_reconciliation() { + let repo = TestRepo::new("orphan-reclaimed-via-public-api"); + let ok_db = || repo.open_db(); + + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) + .expect("the baseline observation should materialize the worktree"); + let worktree_id = baseline.worktree_id.clone(); + + let snapshot = GitSnapshotService::new(&repo.repo_root) + .expect("a snapshot service should construct for the worktree"); + std::fs::write( + repo.repo_root.join("orphan.txt"), + b"never durably referenced", + ) + .expect("the orphan-producing edit should write"); + let x = snapshot.capture_tree().expect("capturing X should succeed"); + snapshot + .pin_tree(&worktree_id, &x) + .expect("pinning X should succeed"); + let x_ref = format!("refs/sce/mutation-cursor/{}/{}", worktree_id.0, x.0); + assert!( + ref_exists(&repo.repo_root, &x_ref), + "the orphan pin must exist before reconciliation runs" + ); + + let outcome = reconcile_worktree(&repo.repo_root, ok_db) + .expect("reconciliation should succeed through the public entrypoint"); + let report = match outcome { + ReconciliationOutcome::Reconciled(report) => report, + ReconciliationOutcome::SkippedNoCheckoutIdentity => { + panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") + } + }; + + assert_eq!( + report.deleted, 1, + "the pass must delete exactly the one pin with no durable root anywhere in the repository" + ); + assert!( + !ref_exists(&repo.repo_root, &x_ref), + "X's snapshot ref must be gone after reconciliation reclaims it" + ); +} + +#[test] +fn current_cursor_pin_survives_reconciliation_without_a_referencing_event_through_the_public_api() { + let repo = TestRepo::new("cursor-retained-no-event"); + let ok_db = || repo.open_db(); + + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) + .expect("the baseline observation should materialize the worktree"); + let worktree_id = baseline.worktree_id.clone(); + let cursor_ref = format!( + "refs/sce/mutation-cursor/{}/{}", + worktree_id.0, baseline.observed_tree.0 + ); + assert!( + ref_exists(&repo.repo_root, &cursor_ref), + "the baseline flush must pin the current cursor tree" + ); + + let db = repo.db(); + let store = MutationTraceStore::new(&db); + let roots = store + .load_tree_roots(&worktree_id) + .expect("load_tree_roots should succeed"); + assert_eq!( + roots.len(), + 1, + "with no event yet, the cursor tree is the worktree's only durable root" + ); + assert!( + roots.contains(&baseline.observed_tree), + "the current cursor tree must be that sole durable root" + ); + + let outcome = reconcile_worktree(&repo.repo_root, ok_db) + .expect("reconciliation should succeed through the public entrypoint"); + let report = match outcome { + ReconciliationOutcome::Reconciled(report) => report, + ReconciliationOutcome::SkippedNoCheckoutIdentity => { + panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") + } + }; + + assert_eq!( + report.deleted, 0, + "the current cursor tree is a durable root via cursor_tree alone and must be retained" + ); + assert!( + ref_exists(&repo.repo_root, &cursor_ref), + "the cursor pin must still resolve after reconciliation" + ); +} + +#[test] +fn historical_before_and_after_pins_survive_reconciliation_after_real_coordinate_transitions() { + let repo = TestRepo::new("historical-retention-real-transitions"); + let ok_db = || repo.open_db(); + + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) + .expect("the baseline observation should materialize the worktree"); + let worktree_id = baseline.worktree_id.clone(); + let tree_a = baseline.observed_tree.clone(); + + let scope = ScopeId("scope-historical-abcd".to_string()); + coordinate( + &repo.repo_root, + &RuntimeBoundary::Start { + scope: scope.clone(), + event: EventId("evt-start".to_string()), + actor_kind: ActorKind::ClaudeCode, + }, + ok_db, + ) + .expect("starting the scope should succeed"); + + std::fs::write(repo.repo_root.join("work.txt"), b"v1").expect("the A -> B edit should write"); + let advance_b = coordinate( + &repo.repo_root, + &RuntimeBoundary::Advance { + scope: scope.clone(), + event: EventId("evt-advance-b".to_string()), + actor_kind: ActorKind::ClaudeCode, + }, + ok_db, + ) + .expect("the A -> B advance should commit one event"); + let tree_b = advance_b.observed_tree.clone(); + + std::fs::write(repo.repo_root.join("work.txt"), b"v2").expect("the B -> C edit should write"); + let advance_c = coordinate( + &repo.repo_root, + &RuntimeBoundary::Advance { + scope: scope.clone(), + event: EventId("evt-advance-c".to_string()), + actor_kind: ActorKind::ClaudeCode, + }, + ok_db, + ) + .expect("the B -> C advance should commit one event"); + let tree_c = advance_c.observed_tree.clone(); + + std::fs::write(repo.repo_root.join("work.txt"), b"v3").expect("the C -> D edit should write"); + let advance_d = coordinate( + &repo.repo_root, + &RuntimeBoundary::Advance { + scope, + event: EventId("evt-advance-d".to_string()), + actor_kind: ActorKind::ClaudeCode, + }, + ok_db, + ) + .expect("the C -> D advance should commit one event"); + let tree_d = advance_d.observed_tree.clone(); + + assert_ne!( + tree_a, tree_b, + "the A -> B advance must observe a real drift" + ); + assert_ne!( + tree_b, tree_c, + "the B -> C advance must observe a real drift" + ); + assert_ne!( + tree_c, tree_d, + "the C -> D advance must observe a real drift" + ); + + let outcome = reconcile_worktree(&repo.repo_root, ok_db) + .expect("reconciliation should succeed through the public entrypoint"); + let report = match outcome { + ReconciliationOutcome::Reconciled(report) => report, + ReconciliationOutcome::SkippedNoCheckoutIdentity => { + panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") + } + }; + assert_eq!( + report.deleted, 0, + "every historical before_tree/after_tree is a durable root: the history A -> B -> C -> D \ + must delete none of {{A, B, C, D}}" + ); + for tree in [&tree_a, &tree_b, &tree_c, &tree_d] { + let tree_ref = format!("refs/sce/mutation-cursor/{}/{}", worktree_id.0, tree.0); + assert!( + ref_exists(&repo.repo_root, &tree_ref), + "the historical pin for {tree:?} must survive reconciliation after the cursor advanced past it" + ); + } +} + +#[test] +fn reconciliation_through_the_public_api_is_idempotent() { + let repo = TestRepo::new("idempotent-via-public-api"); + let ok_db = || repo.open_db(); + + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) + .expect("the baseline observation should materialize the worktree"); + let worktree_id = baseline.worktree_id.clone(); + + let snapshot = GitSnapshotService::new(&repo.repo_root) + .expect("a snapshot service should construct for the worktree"); + std::fs::write(repo.repo_root.join("orphan-1.txt"), b"orphan one") + .expect("the first orphan-producing edit should write"); + let x1 = snapshot + .capture_tree() + .expect("capturing X1 should succeed"); + snapshot + .pin_tree(&worktree_id, &x1) + .expect("pinning X1 should succeed"); + std::fs::write(repo.repo_root.join("orphan-2.txt"), b"orphan two") + .expect("the second orphan-producing edit should write"); + let x2 = snapshot + .capture_tree() + .expect("capturing X2 should succeed"); + snapshot + .pin_tree(&worktree_id, &x2) + .expect("pinning X2 should succeed"); + + let first = + match reconcile_worktree(&repo.repo_root, ok_db).expect("the first pass should succeed") { + ReconciliationOutcome::Reconciled(report) => report, + ReconciliationOutcome::SkippedNoCheckoutIdentity => { + panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") + } + }; + assert_eq!( + first.deleted, 2, + "the first pass must reclaim both orphan pins" + ); + + let second = + match reconcile_worktree(&repo.repo_root, ok_db).expect("the second pass should succeed") { + ReconciliationOutcome::Reconciled(report) => report, + ReconciliationOutcome::SkippedNoCheckoutIdentity => { + panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") + } + }; + assert_eq!( + second.deleted, 0, + "the second pass has nothing left to reclaim" + ); + assert_eq!( + first.local_required, second.local_required, + "the local durable-root count must be stable across both passes" + ); + assert_eq!( + first.retained, second.retained, + "the retained-pin count must be identical once the pass has nothing more to reclaim" + ); +} + +#[test] +fn reconcile_one_linked_worktree_leaves_the_other_worktrees_pins_and_shared_objects_intact() { + let repo = LinkedTestRepo::new("linked-isolation"); + let ok_db = || repo.open_db(); + + let a_baseline = coordinate(&repo.main_root, &RuntimeBoundary::Flush, ok_db) + .expect("A's baseline observation should materialize its worktree"); + let a_id = a_baseline.worktree_id.clone(); + let b_baseline = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, ok_db) + .expect("B's baseline observation should materialize its worktree"); + let b_id = b_baseline.worktree_id.clone(); + assert_ne!( + a_id, b_id, + "A and B must derive distinct checkout identities" + ); + + let b_cursor_ref = format!( + "refs/sce/mutation-cursor/{}/{}", + b_id.0, b_baseline.observed_tree.0 + ); + assert!( + ref_exists(&repo.linked_root, &b_cursor_ref), + "B's own cursor pin must exist before A reconciles" + ); + + let a_snapshot = GitSnapshotService::new(&repo.main_root) + .expect("a snapshot service should construct for A"); + std::fs::write(repo.main_root.join("a-orphan.txt"), b"A's own orphan") + .expect("A's orphan-producing edit should write"); + let a_orphan = a_snapshot + .capture_tree() + .expect("capturing A's orphan tree should succeed"); + a_snapshot + .pin_tree(&a_id, &a_orphan) + .expect("pinning A's orphan should succeed"); + let a_orphan_ref = format!("refs/sce/mutation-cursor/{}/{}", a_id.0, a_orphan.0); + + let outcome = reconcile_worktree(&repo.main_root, ok_db) + .expect("reconciling A should succeed through the public entrypoint"); + let report = match outcome { + ReconciliationOutcome::Reconciled(report) => report, + ReconciliationOutcome::SkippedNoCheckoutIdentity => { + panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") + } + }; + assert_eq!( + report.deleted, 1, + "reconcile_worktree(A) must delete only A's own orphan pin" + ); + assert!( + !ref_exists(&repo.main_root, &a_orphan_ref), + "A's orphan pin must be gone" + ); + + assert!( + ref_exists(&repo.linked_root, &b_cursor_ref), + "reconcile_worktree(A) must never enumerate or delete a refs/sce/mutation-cursor// ref" + ); + let b_snapshot = GitSnapshotService::new(&repo.linked_root) + .expect("a snapshot service should construct for B"); + b_snapshot + .diff_trees(&b_baseline.observed_tree, &b_baseline.observed_tree) + .expect("B's durable tree must still resolve in the shared object database after A's pass"); + + let db = repo.db(); + let store = MutationTraceStore::new(&db); + assert!( + store + .load_worktree(&b_id, None, None) + .expect("loading B's row should succeed") + .is_some(), + "reconcile_worktree(A) must not disturb B's durable worktree row" + ); +} + +#[test] +fn reconcile_a_retains_its_pin_when_another_worktree_durably_requires_the_same_tree() { + let repo = LinkedTestRepo::new("cross-worktree-degraded-retention"); + let ok_db = || repo.open_db(); + + let a_baseline = coordinate(&repo.main_root, &RuntimeBoundary::Flush, ok_db) + .expect("A's baseline observation should materialize its worktree"); + let a_id = a_baseline.worktree_id.clone(); + let b_baseline = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, ok_db) + .expect("B's baseline observation should materialize its worktree"); + let b_id = b_baseline.worktree_id.clone(); + + let b_snapshot = GitSnapshotService::new(&repo.linked_root) + .expect("a snapshot service should construct for B"); + std::fs::write( + repo.linked_root.join("shared-content.txt"), + b"identical content", + ) + .expect("B's edit toward T should write"); + let tree_t = b_snapshot + .capture_tree() + .expect("capturing T should succeed"); + let db = repo.db(); + seed_event(&db, &b_id.0, 1, &b_baseline.observed_tree.0, &tree_t.0); + let b_t_ref = format!("refs/sce/mutation-cursor/{}/{}", b_id.0, tree_t.0); + assert!( + !ref_exists(&repo.linked_root, &b_t_ref), + "B's own pin for T must be deliberately absent -- the degraded state under test" + ); + + let a_snapshot = GitSnapshotService::new(&repo.main_root) + .expect("a snapshot service should construct for A"); + std::fs::write( + repo.main_root.join("shared-content.txt"), + b"identical content", + ) + .expect("A's edit toward the byte-identical T should write"); + let a_tree_t = a_snapshot + .capture_tree() + .expect("capturing A's view of T should succeed"); + assert_eq!( + a_tree_t, tree_t, + "A and B must independently capture byte-identical tree content" + ); + a_snapshot + .pin_tree(&a_id, &a_tree_t) + .expect("A pinning T should succeed"); + let a_t_ref = format!("refs/sce/mutation-cursor/{}/{}", a_id.0, a_tree_t.0); + + let outcome = reconcile_worktree(&repo.main_root, ok_db) + .expect("reconciling A should succeed through the public entrypoint"); + let report = match outcome { + ReconciliationOutcome::Reconciled(report) => report, + ReconciliationOutcome::SkippedNoCheckoutIdentity => { + panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") + } + }; + assert_eq!( + report.deleted, 0, + "A must retain its T pin: T is a repository-wide durable root via B's historical event, \ + even though A does not durably reference T itself" + ); + assert!( + ref_exists(&repo.main_root, &a_t_ref), + "A's T pin is the last SCE ref protecting T and must survive" + ); + run_git(&repo.main_root, &["cat-file", "-t", &a_tree_t.0]); +} + +#[test] +fn missing_local_required_pin_fails_closed_and_deletes_nothing_even_when_another_worktree_pins_the_tree( +) { + let repo = LinkedTestRepo::new("missing-required-pin-fail-closed"); + let ok_db = || repo.open_db(); + + let a_baseline = coordinate(&repo.main_root, &RuntimeBoundary::Flush, ok_db) + .expect("A's baseline observation should materialize its worktree"); + let a_id = a_baseline.worktree_id.clone(); + let tree_a = a_baseline.observed_tree.clone(); + + let a_snapshot = GitSnapshotService::new(&repo.main_root) + .expect("a snapshot service should construct for A"); + std::fs::write(repo.main_root.join("required-but-unpinned.txt"), b"B") + .expect("the edit toward B should write"); + let tree_b = a_snapshot + .capture_tree() + .expect("capturing B should succeed"); + let db = repo.db(); + seed_event(&db, &a_id.0, 1, &tree_a.0, &tree_b.0); + let pin_ref_for_missing_b = format!("refs/sce/mutation-cursor/{}/{}", a_id.0, tree_b.0); + assert!( + !ref_exists(&repo.main_root, &pin_ref_for_missing_b), + "A's pin for B must be deliberately absent" + ); + + std::fs::write(repo.main_root.join("unrelated.txt"), b"X") + .expect("the unrelated edit should write"); + let tree_x = a_snapshot + .capture_tree() + .expect("capturing X should succeed"); + a_snapshot + .pin_tree(&a_id, &tree_x) + .expect("pinning X should succeed"); + let pin_ref_for_x = format!("refs/sce/mutation-cursor/{}/{}", a_id.0, tree_x.0); + + let b_baseline = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, ok_db) + .expect("the linked worktree's baseline observation should materialize"); + let b_id = b_baseline.worktree_id.clone(); + let b_snapshot = GitSnapshotService::new(&repo.linked_root) + .expect("a snapshot service should construct for the linked worktree"); + b_snapshot + .pin_tree(&b_id, &tree_b) + .expect("the other worktree pinning B should succeed"); + + let outcome = reconcile_worktree(&repo.main_root, ok_db); + match outcome { + Err(ReconcileError::MissingRequiredPins { missing }) => { + assert_eq!( + missing, + vec![tree_b.clone()], + "the missing pin must name exactly B" + ); + } + other => panic!("expected MissingRequiredPins naming B, got {other:?}"), + } + + assert!( + ref_exists(&repo.main_root, &pin_ref_for_x), + "A's X pin must be left untouched by a fail-closed pass" + ); + assert!( + !ref_exists(&repo.main_root, &pin_ref_for_missing_b), + "A's still-absent pin for B must remain absent -- reconciliation never creates or repairs a pin" + ); +} + +#[test] +fn a_malformed_namespace_ref_fails_closed_through_the_public_entrypoint() { + let repo = TestRepo::new("malformed-ref-fail-closed"); + let ok_db = || repo.open_db(); + + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) + .expect("the baseline observation should materialize the worktree"); + let worktree_id = baseline.worktree_id.clone(); + let cursor_ref = format!( + "refs/sce/mutation-cursor/{}/{}", + worktree_id.0, baseline.observed_tree.0 + ); + + let snapshot = GitSnapshotService::new(&repo.repo_root) + .expect("a snapshot service should construct for the worktree"); + std::fs::write(repo.repo_root.join("unrelated.txt"), b"unrelated content") + .expect("the unrelated edit should write"); + let some_tree = snapshot + .capture_tree() + .expect("capturing an arbitrary tree should succeed"); + let symbolic_ref = format!("refs/sce/mutation-cursor/{}/{}", worktree_id.0, some_tree.0); + run_git( + &repo.repo_root, + &["symbolic-ref", &symbolic_ref, &cursor_ref], + ); + + let outcome = reconcile_worktree(&repo.repo_root, ok_db); + match outcome { + Err(ReconcileError::MalformedPin { ref_name, .. }) => { + assert_eq!(ref_name, symbolic_ref); + } + other => panic!("expected MalformedPin for the symbolic ref, got {other:?}"), + } + + assert!( + ref_exists(&repo.repo_root, &cursor_ref), + "the pass must fail closed before deleting anything, including the well-formed cursor pin" + ); + assert_eq!( + run_git(&repo.repo_root, &["symbolic-ref", &symbolic_ref]).trim(), + cursor_ref, + "the malformed symbolic ref itself must be left untouched" + ); +} + +#[test] +fn reconciliation_makes_no_protocol_or_marker_write() { + let repo = TestRepo::new("no-protocol-or-marker-write"); + let ok_db = || repo.open_db(); + let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); + let marker = ExternalTaintMarker::new(&git_dir); + + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) + .expect("the baseline observation should materialize the worktree"); + let worktree_id = baseline.worktree_id.clone(); + + let snapshot = GitSnapshotService::new(&repo.repo_root) + .expect("a snapshot service should construct for the worktree"); + std::fs::write(repo.repo_root.join("orphan.txt"), b"orphan") + .expect("the orphan-producing edit should write"); + let orphan = snapshot + .capture_tree() + .expect("capturing the orphan tree should succeed"); + snapshot + .pin_tree(&worktree_id, &orphan) + .expect("pinning the orphan should succeed"); + + let tables = [ + "mutation_trace_worktrees", + "mutation_trace_scopes", + "mutation_trace_events", + "mutation_trace_processed_events", + "mutation_trace_event_active_scopes", + ]; + let db_before = repo.db(); + let counts_before: Vec = tables + .iter() + .map(|table| row_count(&db_before, table)) + .collect(); + let store_before = MutationTraceStore::new(&db_before); + let worktree_state_before = store_before + .load_worktree(&worktree_id, None, None) + .expect("loading the worktree row should succeed") + .expect("the worktree row should exist") + .worktree_state; + let marker_existed_before = marker + .exists() + .expect("marker existence should resolve before the pass"); + assert!( + !marker_existed_before, + "a successful coordinate() must have already cleared the marker before reconciliation runs" + ); + + let outcome = reconcile_worktree(&repo.repo_root, ok_db) + .expect("reconciliation should succeed through the public entrypoint"); + let report = match outcome { + ReconciliationOutcome::Reconciled(report) => report, + ReconciliationOutcome::SkippedNoCheckoutIdentity => { + panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") + } + }; + assert_eq!( + report.deleted, 1, + "the pass must actually mutate Git refs, not be a trivial no-op" + ); + + let db_after = repo.db(); + let counts_after: Vec = tables + .iter() + .map(|table| row_count(&db_after, table)) + .collect(); + assert_eq!( + counts_before, counts_after, + "every mutation-trace table's row count must be byte-identical after a reconciliation pass" + ); + let store_after = MutationTraceStore::new(&db_after); + let worktree_state_after = store_after + .load_worktree(&worktree_id, None, None) + .expect("loading the worktree row should succeed") + .expect("the worktree row should exist") + .worktree_state; + assert_eq!( + worktree_state_before, worktree_state_after, + "cursor_tree/revision/tainted/failure_kind/needs_rebaseline must be unchanged by reconciliation" + ); + assert!( + !marker + .exists() + .expect("marker existence should resolve after the pass"), + "reconciliation must not create the external-taint marker" + ); +} + +#[test] +fn reconciliation_deletes_a_stale_ref_without_reclaiming_the_object_through_the_public_api() { + let repo = TestRepo::new("no-object-reclamation-via-public-api"); + let ok_db = || repo.open_db(); + + let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) + .expect("the baseline observation should materialize the worktree"); + let worktree_id = baseline.worktree_id.clone(); + + let snapshot = GitSnapshotService::new(&repo.repo_root) + .expect("a snapshot service should construct for the worktree"); + std::fs::write( + repo.repo_root + .join("only-reachable-through-a-stale-pin.txt"), + b"O", + ) + .expect("the edit toward O should write"); + let tree_o = snapshot.capture_tree().expect("capturing O should succeed"); + snapshot + .pin_tree(&worktree_id, &tree_o) + .expect("pinning O should succeed"); + let o_ref = format!("refs/sce/mutation-cursor/{}/{}", worktree_id.0, tree_o.0); + + run_git(&repo.repo_root, &["cat-file", "-t", &tree_o.0]); + + let outcome = reconcile_worktree(&repo.repo_root, ok_db) + .expect("reconciliation should succeed through the public entrypoint"); + let report = match outcome { + ReconciliationOutcome::Reconciled(report) => report, + ReconciliationOutcome::SkippedNoCheckoutIdentity => { + panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") + } + }; + assert_eq!( + report.deleted, 1, + "O's only SCE ref must be reclaimed as an orphan pin" + ); + assert!( + !ref_exists(&repo.repo_root, &o_ref), + "O's ref must be gone after reconciliation" + ); + + let cat_file_type = run_git(&repo.repo_root, &["cat-file", "-t", &tree_o.0]) + .trim() + .to_string(); + assert_eq!( + cat_file_type, "tree", + "O must still resolve via git cat-file -t immediately after its stale ref is deleted, \ + because reconciliation runs no git gc / git prune / git reflog expire" + ); +} + +#[test] +fn missing_checkout_identity_through_the_public_entrypoint_returns_skipped_outcome() { + let repo = TestRepo::new("missing-checkout-identity-public-entrypoint"); + + let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); + assert!( + read_checkout_id(&git_dir) + .expect("reading the checkout-id file should succeed") + .is_none(), + "a freshly initialized repository must have no checkout identity yet" + ); + + let outcome = reconcile_worktree(&repo.repo_root, || { + panic!("open_db must never be called on the missing-checkout-identity path") + }) + .expect("the missing-checkout-identity path is a clean Ok(..), never an Err"); + + assert_eq!( + outcome, + ReconciliationOutcome::SkippedNoCheckoutIdentity, + "with no current checkout identity to derive an owned namespace from, reconciliation must \ + return the distinct skip outcome, never a zero-count Reconciled(..) report" + ); + assert!( + read_checkout_id(&git_dir) + .expect("reading the checkout-id file should succeed") + .is_none(), + "the skip must not create a checkout identity" ); - cleanup(&repo_root); } diff --git a/cli/src/services/mutation_trace/store.rs b/cli/src/services/mutation_trace/store.rs index 68953e15..9b2c06b0 100644 --- a/cli/src/services/mutation_trace/store.rs +++ b/cli/src/services/mutation_trace/store.rs @@ -201,6 +201,24 @@ const SELECT_MUTATION_EVENT_SQL: &str = "SELECT before_tree, after_tree, tainted FROM mutation_trace_events WHERE worktree_id = ?1 AND revision = ?2"; const SELECT_MUTATION_EVENT_ACTIVE_SCOPES_SQL: &str = "SELECT scope_id FROM mutation_trace_event_active_scopes WHERE worktree_id = ?1 AND revision = ?2"; +/// One worktree's complete durable tree root set — its cursor tree plus the +/// `before_tree` / `after_tree` of every historical `mutation_trace_events` +/// row — as a single `UNION` statement so the whole set is read from one +/// database snapshot, never assembled from independent `SELECT`s. +const SELECT_TREE_ROOTS_BY_WORKTREE_SQL: &str = + "SELECT cursor_tree AS tree FROM mutation_trace_worktrees WHERE worktree_id = ?1 + UNION + SELECT before_tree AS tree FROM mutation_trace_events WHERE worktree_id = ?1 + UNION + SELECT after_tree AS tree FROM mutation_trace_events WHERE worktree_id = ?1"; +/// The same three `TreeId` columns unioned across **every** worktree in the +/// repository, in one statement / one snapshot — the reconciler's +/// repository-wide retention set. +const SELECT_ALL_TREE_ROOTS_SQL: &str = "SELECT cursor_tree AS tree FROM mutation_trace_worktrees + UNION + SELECT before_tree AS tree FROM mutation_trace_events + UNION + SELECT after_tree AS tree FROM mutation_trace_events"; /// Idle-insert: only takes effect when `worktree_id` has no row yet, so an /// existing worktree's cursor/revision/failure state is never overwritten. const INSERT_WORKTREE_IF_ABSENT_SQL: &str = "INSERT INTO mutation_trace_worktrees @@ -650,6 +668,58 @@ impl<'a> MutationTraceStore<'a> { })) } + /// Reads `worktree`'s complete durable tree root set: its + /// `mutation_trace_worktrees.cursor_tree`, plus the `before_tree` and + /// `after_tree` of every `mutation_trace_events` row for `worktree`, + /// deduplicated. Returns an empty set (not an error) when `worktree` has + /// no durable row at all. + /// + /// Read-only, cold path — never called from `load_worktree` or any + /// hook-boundary path, exactly like [`MutationTraceStore::load_mutation_event`]. + /// It reads only the three `TreeId` columns above: never + /// `mutation_trace_scopes` / `mutation_trace_processed_events` / + /// `mutation_trace_event_active_scopes`, never another worktree's trees, + /// and never transient `AttemptState` / `external_taint`. + /// + /// The whole set is produced by **one** SQL statement (a `UNION` of the + /// three columns) through **one** `query_map` call, so a concurrent + /// mutation-cursor commit — which atomically moves `cursor_tree` from `T` + /// to `X` and inserts `MutationEvent { before_tree = T, after_tree = X }` + /// in the same transaction — cannot expose a torn root set that omits `T`: + /// the single statement observes either the pre-commit snapshot + /// (`cursor_tree` still contains `T`) or the post-commit snapshot + /// (`before_tree` contains `T`). + pub fn load_tree_roots(&self, worktree: &WorktreeId) -> Result> { + let rows = self.db.query_map( + SELECT_TREE_ROOTS_BY_WORKTREE_SQL, + (worktree.0.as_str(),), + tree_root_row_from_turso, + )?; + + Ok(rows.into_iter().collect()) + } + + /// Reads the repository-wide durable tree root set: the union of + /// `mutation_trace_worktrees.cursor_tree`, `mutation_trace_events.before_tree`, + /// and `mutation_trace_events.after_tree` across **every** worktree, + /// deduplicated. Returns an empty set (not an error) for a repository with + /// no mutation-cursor rows. + /// + /// This is the reconciler's retention set: linked worktrees share one Git + /// object database, so a ref owned by worktree `A` may be the last SCE ref + /// protecting a tree that only worktree `B` durably requires. Read-only, + /// cold path, and — like [`MutationTraceStore::load_tree_roots`] — one SQL + /// statement through one `query_map` call, so it cannot tear across a + /// concurrent atomic `cursor T -> X` + `event T -> X` commit on another + /// worktree. + pub fn load_all_tree_roots(&self) -> Result> { + let rows = self + .db + .query_map(SELECT_ALL_TREE_ROOTS_SQL, (), tree_root_row_from_turso)?; + + Ok(rows.into_iter().collect()) + } + fn load_worktree_state(&self, worktree: &WorktreeId) -> Result> { let rows = self.db.query_map( SELECT_WORKTREE_SQL, @@ -844,6 +914,13 @@ fn effective_referenced_scope<'k>( } } +fn tree_root_row_from_turso(row: &turso::Row) -> Result { + let tree: String = row + .get(0) + .context("failed to read a durable tree root column")?; + Ok(TreeId(tree)) +} + fn worktree_state_row_from_turso(row: &turso::Row) -> Result { let cursor_tree: String = row .get(0) @@ -999,7 +1076,6 @@ fn reconstruct_boundary( #[cfg(test)] mod tests { - use std::sync::atomic::{AtomicU64, Ordering}; use std::thread; use super::*; @@ -1122,21 +1198,28 @@ mod tests { assert!(decode_boundary_kind("unknown").is_err()); } - static NEXT_TEST_DB_ID: AtomicU64 = AtomicU64::new(0); + struct TestDbPath { + _temp_dir: tempfile::TempDir, + path: std::path::PathBuf, + } - fn unique_test_db_path(label: &str) -> std::path::PathBuf { - let id = NEXT_TEST_DB_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir() - .join(format!( - "sce-mutation-trace-store-{label}-{}-{id}", - std::process::id() - )) - .join("agent-trace.db") + impl TestDbPath { + fn path(&self) -> &std::path::Path { + &self.path + } } - fn remove_test_db(db_path: &std::path::Path) { - if let Some(parent) = db_path.parent() { - std::fs::remove_dir_all(parent).expect("test DB directory should be removed"); + fn test_db_path(label: &str) -> TestDbPath { + let temp_dir = tempfile::Builder::new() + .prefix(&format!("sce-mutation-trace-store-{label}-")) + .tempdir() + .expect("test temp directory should be created"); + + let path = temp_dir.path().join("agent-trace.db"); + + TestDbPath { + _temp_dir: temp_dir, + path, } } @@ -1233,22 +1316,22 @@ mod tests { #[test] fn load_worktree_returns_none_for_a_missing_worktree() { - let db_path = unique_test_db_path("missing-worktree"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("missing-worktree"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); let projection = store .load_worktree(&WorktreeId("wt-missing".to_string()), None, None) .expect("load_worktree should succeed"); assert!(projection.is_none()); - - remove_test_db(&db_path); } #[test] fn load_worktree_with_no_scope_or_event_key_loads_only_active_scopes() { - let db_path = unique_test_db_path("case-1-active-only"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("case-1-active-only"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 5); @@ -1267,14 +1350,13 @@ mod tests { vec![&ScopeId("scope-active".to_string())] ); assert!(projection.processed_events.is_empty()); - - remove_test_db(&db_path); } #[test] fn load_worktree_with_explicit_scope_includes_it_regardless_of_status() { - let db_path = unique_test_db_path("case-2-explicit-scope"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("case-2-explicit-scope"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1297,14 +1379,13 @@ mod tests { worktree_id: WorktreeId("wt-1".to_string()), }) ); - - remove_test_db(&db_path); } #[test] fn load_worktree_with_explicit_scope_on_another_worktree_errors() { - let db_path = unique_test_db_path("case-2-wrong-worktree"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("case-2-wrong-worktree"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1319,14 +1400,13 @@ mod tests { ) .expect_err("scope belonging to another worktree should error"); assert!(error.to_string().contains("scope-1")); - - remove_test_db(&db_path); } #[test] fn load_worktree_with_explicit_missing_scope_errors() { - let db_path = unique_test_db_path("case-2-missing-scope"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("case-2-missing-scope"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1339,14 +1419,13 @@ mod tests { ) .expect_err("missing effective scope should error"); assert!(error.to_string().contains("scope-missing")); - - remove_test_db(&db_path); } #[test] fn load_worktree_with_only_event_key_loads_its_scope_and_replay_row() { - let db_path = unique_test_db_path("case-3-event-key-only"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("case-3-event-key-only"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1374,14 +1453,13 @@ mod tests { projection.processed_events, [event_key].into_iter().collect() ); - - remove_test_db(&db_path); } #[test] fn load_worktree_with_event_key_scope_on_another_worktree_errors() { - let db_path = unique_test_db_path("case-3-wrong-worktree"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("case-3-wrong-worktree"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1397,14 +1475,13 @@ mod tests { .load_worktree(&WorktreeId("wt-1".to_string()), None, Some(&event_key)) .expect_err("event_key scope on another worktree should error"); assert!(error.to_string().contains("scope-1")); - - remove_test_db(&db_path); } #[test] fn load_worktree_with_event_key_missing_scope_errors() { - let db_path = unique_test_db_path("case-3-missing-scope"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("case-3-missing-scope"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1418,14 +1495,13 @@ mod tests { .load_worktree(&WorktreeId("wt-1".to_string()), None, Some(&event_key)) .expect_err("missing event_key.scope_id should error"); assert!(error.to_string().contains("scope-missing")); - - remove_test_db(&db_path); } #[test] fn load_worktree_with_event_key_missing_scope_and_orphan_replay_row_errors() { - let db_path = unique_test_db_path("case-3-orphan-replay-row"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("case-3-orphan-replay-row"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1442,14 +1518,13 @@ mod tests { "an orphan processed-event row must not let a missing scope produce a projection", ); assert!(error.to_string().contains("scope-missing")); - - remove_test_db(&db_path); } #[test] fn load_worktree_with_agreeing_scope_and_event_key_loads_it_once() { - let db_path = unique_test_db_path("case-4-agreeing"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("case-4-agreeing"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1473,14 +1548,13 @@ mod tests { assert!(projection .scopes .contains_key(&ScopeId("scope-1".to_string()))); - - remove_test_db(&db_path); } #[test] fn load_worktree_with_disagreeing_scope_and_event_key_errors_without_loading() { - let db_path = unique_test_db_path("case-5-disagreeing"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("case-5-disagreeing"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1501,28 +1575,26 @@ mod tests { .expect_err("disagreeing scope/event_key.scope_id should error"); assert!(error.to_string().contains("scope-a")); assert!(error.to_string().contains("scope-b")); - - remove_test_db(&db_path); } #[test] fn load_mutation_event_returns_none_when_missing() { - let db_path = unique_test_db_path("cold-path-missing"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("cold-path-missing"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); let event = store .load_mutation_event(&WorktreeId("wt-1".to_string()), 1) .expect("load_mutation_event should succeed"); assert!(event.is_none()); - - remove_test_db(&db_path); } #[test] fn load_mutation_event_reconstructs_ai_exclusive_start_event() { - let db_path = unique_test_db_path("cold-path-ai-exclusive-start"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("cold-path-ai-exclusive-start"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_mutation_event( @@ -1561,14 +1633,13 @@ mod tests { }, } ); - - remove_test_db(&db_path); } #[test] fn load_mutation_event_reconstructs_a_flush_event_with_multiple_active_scopes() { - let db_path = unique_test_db_path("cold-path-flush"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("cold-path-flush"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_mutation_event( @@ -1611,14 +1682,13 @@ mod tests { }, } ); - - remove_test_db(&db_path); } #[test] fn into_protocol_state_carries_only_the_loaded_worktree_and_leaves_transient_fields_empty() { - let db_path = unique_test_db_path("into-protocol-state"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("into-protocol-state"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 7); @@ -1642,14 +1712,13 @@ mod tests { assert!(protocol_state.attempts.is_empty()); assert!(protocol_state.mutation_events.is_empty()); assert!(protocol_state.external_taint.is_empty()); - - remove_test_db(&db_path); } #[test] fn initialize_worktree_inserts_a_fresh_healthy_cursor() { - let db_path = unique_test_db_path("init-worktree-fresh"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("init-worktree-fresh"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); store @@ -1674,14 +1743,13 @@ mod tests { needs_rebaseline: false, } ); - - remove_test_db(&db_path); } #[test] fn initialize_worktree_never_overwrites_an_existing_cursor() { - let db_path = unique_test_db_path("init-worktree-idempotent"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("init-worktree-idempotent"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 5); @@ -1703,14 +1771,13 @@ mod tests { TreeId("tree-0".to_string()) ); assert_eq!(projection.worktree_state.revision, 5); - - remove_test_db(&db_path); } #[test] fn register_scope_inserts_never_seen_when_missing() { - let db_path = unique_test_db_path("register-scope-fresh"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("register-scope-fresh"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1731,14 +1798,13 @@ mod tests { worktree_id: WorktreeId("wt-1".to_string()), } ); - - remove_test_db(&db_path); } #[test] fn register_scope_returns_existing_state_when_worktree_and_actor_match() { - let db_path = unique_test_db_path("register-scope-existing-match"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("register-scope-existing-match"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1760,14 +1826,13 @@ mod tests { worktree_id: WorktreeId("wt-1".to_string()), } ); - - remove_test_db(&db_path); } #[test] fn register_scope_errors_on_worktree_mismatch() { - let db_path = unique_test_db_path("register-scope-worktree-mismatch"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("register-scope-worktree-mismatch"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1782,14 +1847,13 @@ mod tests { ) .expect_err("a worktree mismatch on an existing scope should error"); assert!(error.to_string().contains("scope-1")); - - remove_test_db(&db_path); } #[test] fn register_scope_errors_on_actor_mismatch() { - let db_path = unique_test_db_path("register-scope-actor-mismatch"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("register-scope-actor-mismatch"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_worktree(&db, "wt-1", 0); @@ -1803,14 +1867,13 @@ mod tests { ) .expect_err("an actor mismatch on an existing scope should error"); assert!(error.to_string().contains("scope-1")); - - remove_test_db(&db_path); } #[test] fn register_scope_errors_when_worktree_does_not_exist_and_leaves_no_scope_row() { - let db_path = unique_test_db_path("register-scope-missing-worktree-fresh"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("register-scope-missing-worktree-fresh"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); let error = store @@ -1830,14 +1893,13 @@ mod tests { scope_state.is_none(), "a failed register_scope must not leave an orphan scope row" ); - - remove_test_db(&db_path); } #[test] fn register_scope_errors_when_existing_scopes_worktree_row_is_missing() { - let db_path = unique_test_db_path("register-scope-missing-worktree-existing"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("register-scope-missing-worktree-existing"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); insert_scope(&db, "scope-1", "wt-missing", ScopeStatus::Active); @@ -1853,8 +1915,6 @@ mod tests { ); assert!(error.to_string().contains("scope-1")); assert!(error.to_string().contains("wt-missing")); - - remove_test_db(&db_path); } fn healthy_worktree_state(revision: u64) -> WorktreeState { @@ -2323,8 +2383,9 @@ mod tests { #[test] fn commit_applies_a_full_transition_and_makes_every_write_visible() { - let db_path = unique_test_db_path("commit-applies-full-transition"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("commit-applies-full-transition"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); let wt = WorktreeId("wt0".to_string()); let scope_id = ScopeId("scope0".to_string()); @@ -2407,14 +2468,13 @@ mod tests { .expect("mutation-event read should succeed") .expect("mutation-event row should exist"); assert_eq!(reloaded_event, mutation_event); - - remove_test_db(&db_path); } #[test] fn commit_returns_conflict_and_writes_nothing_when_the_worktree_revision_has_moved_on() { - let db_path = unique_test_db_path("commit-conflict"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("commit-conflict"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); let wt = WorktreeId("wt0".to_string()); insert_worktree(&db, &wt.0, 5); @@ -2438,14 +2498,13 @@ mod tests { .expect("worktree read should succeed") .expect("worktree row should exist"); assert_eq!(worktree_state.revision, 5); - - remove_test_db(&db_path); } #[test] fn commit_propagates_a_deterministic_failure_without_reporting_conflict() { - let db_path = unique_test_db_path("commit-deterministic-failure"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("commit-deterministic-failure"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); let wt = WorktreeId("wt0".to_string()); let scope_id = ScopeId("scope0".to_string()); @@ -2483,8 +2542,6 @@ mod tests { worktree_state.revision, 0, "the guard's own revision advance must roll back together with the failed insert" ); - - remove_test_db(&db_path); } struct RaceEvidence { @@ -2608,13 +2665,14 @@ mod tests { #[test] fn commit_from_two_independent_connections_races_and_only_one_applies() { - let db_path = unique_test_db_path("commit-two-writer-race"); + let db_fixture = test_db_path("commit-two-writer-race"); + let db_path = db_fixture.path(); let wt = WorktreeId("wt0".to_string()); let scope_a = ScopeId("scope-a".to_string()); let scope_b = ScopeId("scope-b".to_string()); { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); insert_worktree(&db, &wt.0, 0); insert_scope(&db, &scope_a.0, &wt.0, ScopeStatus::Active); insert_scope(&db, &scope_b.0, &wt.0, ScopeStatus::Active); @@ -2646,9 +2704,9 @@ mod tests { let transition_a = closing_transition(&before, &wt, &writer_a); let transition_b = closing_transition(&before, &wt, &writer_b); - let db_a = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) + let db_a = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path) .expect("writer A handle should open"); - let db_b = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) + let db_b = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path) .expect("writer B handle should open"); let handle_a = thread::spawn(move || MutationTraceStore::new(&db_a).commit(&transition_a)); @@ -2678,7 +2736,7 @@ mod tests { "exactly one writer should conflict from the same starting revision: {results:?}" ); - let db_reopened = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) + let db_reopened = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path) .expect("reopened handle should open"); let store = MutationTraceStore::new(&db_reopened); @@ -2697,8 +2755,6 @@ mod tests { .expect("exactly one writer's mutation event should be visible at the new revision"); assert_race_winner_state(&store, &wt, &persisted_event, &writer_a, &writer_b); - - remove_test_db(&db_path); } fn assert_atomic_rollback_state( @@ -2774,8 +2830,9 @@ mod tests { #[test] fn commit_rolls_back_every_write_kind_together_on_a_deterministic_failure() { - let db_path = unique_test_db_path("commit-atomic-rollback"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("commit-atomic-rollback"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); let wt = WorktreeId("wt0".to_string()); let scope_id = ScopeId("scope0".to_string()); @@ -2825,14 +2882,13 @@ mod tests { assert!(error.to_string().contains("execute failed")); assert_atomic_rollback_state(&store, &wt, &scope_id, &scope_a, &scope_z); - - remove_test_db(&db_path); } #[test] fn commit_round_trips_u64_max_through_the_real_database() { - let db_path = unique_test_db_path("commit-u64-max"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("commit-u64-max"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); let wt = WorktreeId("wt0".to_string()); insert_worktree(&db, &wt.0, u64::MAX - 1); @@ -2856,14 +2912,13 @@ mod tests { .expect("worktree read should succeed") .expect("worktree row should exist"); assert_eq!(worktree_state.revision, u64::MAX); - - remove_test_db(&db_path); } #[test] fn commit_rejects_a_replayed_event_key_via_the_processed_event_uniqueness_constraint() { - let db_path = unique_test_db_path("commit-replay-uniqueness"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("commit-replay-uniqueness"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); let wt = WorktreeId("wt0".to_string()); let scope_id = ScopeId("scope0".to_string()); @@ -2902,14 +2957,13 @@ mod tests { worktree_state.revision, 0, "the whole transaction must roll back on a replay rejection" ); - - remove_test_db(&db_path); } #[test] fn commit_of_strong_recovery_abandons_every_live_scope_on_the_worktree() { - let db_path = unique_test_db_path("commit-strong-recovery"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("commit-strong-recovery"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); let wt = WorktreeId("wt0".to_string()); let scope_a = ScopeId("scope-a".to_string()); @@ -2961,14 +3015,13 @@ mod tests { .expect("scope row should exist"); assert_eq!(scope_state.status, ScopeStatus::Abandoned); } - - remove_test_db(&db_path); } #[test] fn commit_of_needs_only_recovery_leaves_live_scopes_active() { - let db_path = unique_test_db_path("commit-needs-only-recovery"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db_fixture = test_db_path("commit-needs-only-recovery"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); let store = MutationTraceStore::new(&db); let wt = WorktreeId("wt0".to_string()); let scope_id = ScopeId("scope0".to_string()); @@ -3018,8 +3071,6 @@ mod tests { ScopeStatus::Active, "a live scope must survive needs-only recovery untouched" ); - - remove_test_db(&db_path); } fn insert_worktree_with_state( @@ -3147,16 +3198,17 @@ mod tests { #[test] fn round_trip_start_persists_and_reloads_exactly_after_reopening_the_database() { - let db_path = unique_test_db_path("roundtrip-start"); + let db_fixture = test_db_path("roundtrip-start"); + let db_path = db_fixture.path(); let wt = WorktreeId("wt0".to_string()); let scope_id = ScopeId("scope0".to_string()); { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); insert_worktree(&db, &wt.0, 0); insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::NeverSeen); } - let before = load_before_state(&db_path, &wt, Some(&scope_id), None); + let before = load_before_state(db_path, &wt, Some(&scope_id), None); let attempt = AttemptId("attempt0".to_string()); let event_id = EventId("event0".to_string()); let prepared = prepare( @@ -3178,29 +3230,28 @@ mod tests { let transition = DurableTransition::between(&before, &after, &wt) .expect("between should succeed") .expect("a start transition should produce a durable transition"); - assert_eq!(commit_transition(&db_path, &transition), CasResult::Applied); + assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); let event_key = EventKey { scope_id: scope_id.clone(), event_id, }; - assert_round_trip(&db_path, &wt, Some(&scope_id), Some(&event_key), &after); - - remove_test_db(&db_path); + assert_round_trip(db_path, &wt, Some(&scope_id), Some(&event_key), &after); } #[test] fn round_trip_advance_persists_and_reloads_exactly_after_reopening_the_database() { - let db_path = unique_test_db_path("roundtrip-advance"); + let db_fixture = test_db_path("roundtrip-advance"); + let db_path = db_fixture.path(); let wt = WorktreeId("wt0".to_string()); let scope_id = ScopeId("scope0".to_string()); { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); insert_worktree(&db, &wt.0, 0); insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::Active); } - let before = load_before_state(&db_path, &wt, Some(&scope_id), None); + let before = load_before_state(db_path, &wt, Some(&scope_id), None); let attempt = AttemptId("attempt0".to_string()); let event_id = EventId("event0".to_string()); let prepared = prepare( @@ -3222,29 +3273,28 @@ mod tests { let transition = DurableTransition::between(&before, &after, &wt) .expect("between should succeed") .expect("an advance transition should produce a durable transition"); - assert_eq!(commit_transition(&db_path, &transition), CasResult::Applied); + assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); let event_key = EventKey { scope_id: scope_id.clone(), event_id, }; - assert_round_trip(&db_path, &wt, Some(&scope_id), Some(&event_key), &after); - - remove_test_db(&db_path); + assert_round_trip(db_path, &wt, Some(&scope_id), Some(&event_key), &after); } #[test] fn round_trip_close_persists_and_reloads_exactly_after_reopening_the_database() { - let db_path = unique_test_db_path("roundtrip-close"); + let db_fixture = test_db_path("roundtrip-close"); + let db_path = db_fixture.path(); let wt = WorktreeId("wt0".to_string()); let scope_id = ScopeId("scope0".to_string()); { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); insert_worktree(&db, &wt.0, 0); insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::Active); } - let before = load_before_state(&db_path, &wt, Some(&scope_id), None); + let before = load_before_state(db_path, &wt, Some(&scope_id), None); let attempt = AttemptId("attempt0".to_string()); let event_id = EventId("event0".to_string()); let prepared = prepare( @@ -3270,27 +3320,26 @@ mod tests { let transition = DurableTransition::between(&before, &after, &wt) .expect("between should succeed") .expect("a close transition should produce a durable transition"); - assert_eq!(commit_transition(&db_path, &transition), CasResult::Applied); + assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); let event_key = EventKey { scope_id: scope_id.clone(), event_id, }; - assert_round_trip(&db_path, &wt, Some(&scope_id), Some(&event_key), &after); - - remove_test_db(&db_path); + assert_round_trip(db_path, &wt, Some(&scope_id), Some(&event_key), &after); } #[test] fn round_trip_flush_with_change_persists_and_reloads_exactly_after_reopening_the_database() { - let db_path = unique_test_db_path("roundtrip-flush-change"); + let db_fixture = test_db_path("roundtrip-flush-change"); + let db_path = db_fixture.path(); let wt = WorktreeId("wt0".to_string()); { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); insert_worktree(&db, &wt.0, 0); } - let before = load_before_state(&db_path, &wt, None, None); + let before = load_before_state(db_path, &wt, None, None); let attempt = AttemptId("attempt0".to_string()); let prepared = prepare( &before, @@ -3310,23 +3359,22 @@ mod tests { let transition = DurableTransition::between(&before, &after, &wt) .expect("between should succeed") .expect("a changed flush transition should produce a durable transition"); - assert_eq!(commit_transition(&db_path, &transition), CasResult::Applied); - - assert_round_trip(&db_path, &wt, None, None, &after); + assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - remove_test_db(&db_path); + assert_round_trip(db_path, &wt, None, None, &after); } #[test] fn round_trip_flush_without_change_persists_nothing_new() { - let db_path = unique_test_db_path("roundtrip-flush-no-change"); + let db_fixture = test_db_path("roundtrip-flush-no-change"); + let db_path = db_fixture.path(); let wt = WorktreeId("wt0".to_string()); { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); insert_worktree(&db, &wt.0, 0); } - let before = load_before_state(&db_path, &wt, None, None); + let before = load_before_state(db_path, &wt, None, None); let attempt = AttemptId("attempt0".to_string()); let prepared = prepare( &before, @@ -3349,81 +3397,78 @@ mod tests { "a no-change flush must produce no durable transition to persist" ); - assert_round_trip(&db_path, &wt, None, None, &after); - - remove_test_db(&db_path); + assert_round_trip(db_path, &wt, None, None, &after); } #[test] fn round_trip_taint_persists_and_reloads_exactly_after_reopening_the_database() { - let db_path = unique_test_db_path("roundtrip-taint"); + let db_fixture = test_db_path("roundtrip-taint"); + let db_path = db_fixture.path(); let wt = WorktreeId("wt0".to_string()); { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); insert_worktree(&db, &wt.0, 0); } - let before = load_before_state(&db_path, &wt, None, None); + let before = load_before_state(db_path, &wt, None, None); let after = taint(&before, &wt); let transition = DurableTransition::between(&before, &after, &wt) .expect("between should succeed") .expect("taint should produce a durable transition"); - assert_eq!(commit_transition(&db_path, &transition), CasResult::Applied); - - assert_round_trip(&db_path, &wt, None, None, &after); + assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - remove_test_db(&db_path); + assert_round_trip(db_path, &wt, None, None, &after); } #[test] fn round_trip_abandon_persists_and_reloads_exactly_after_reopening_the_database() { - let db_path = unique_test_db_path("roundtrip-abandon"); + let db_fixture = test_db_path("roundtrip-abandon"); + let db_path = db_fixture.path(); let wt = WorktreeId("wt0".to_string()); let scope_id = ScopeId("scope0".to_string()); { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); insert_worktree(&db, &wt.0, 0); insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::Active); } - let before = load_before_state(&db_path, &wt, Some(&scope_id), None); + let before = load_before_state(db_path, &wt, Some(&scope_id), None); let after = abandon(&before, &scope_id); let transition = DurableTransition::between(&before, &after, &wt) .expect("between should succeed") .expect("abandon should produce a durable transition"); - assert_eq!(commit_transition(&db_path, &transition), CasResult::Applied); + assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - assert_round_trip(&db_path, &wt, Some(&scope_id), None, &after); - - remove_test_db(&db_path); + assert_round_trip(db_path, &wt, Some(&scope_id), None, &after); } #[test] fn round_trip_strong_recovery_abandons_every_live_scope_after_reopening_the_database() { - let db_path = unique_test_db_path("roundtrip-recover-strong"); + let db_fixture = test_db_path("roundtrip-recover-strong"); + let db_path = db_fixture.path(); let wt = WorktreeId("wt0".to_string()); let scope_a = ScopeId("scope-a".to_string()); let scope_b = ScopeId("scope-b".to_string()); { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); insert_worktree_with_state(&db, &wt.0, 0, true, FailureKind::SnapshotFailure, false); insert_scope(&db, &scope_a.0, &wt.0, ScopeStatus::Active); insert_scope(&db, &scope_b.0, &wt.0, ScopeStatus::Active); } - let before = load_before_state(&db_path, &wt, None, None); + let before = load_before_state(db_path, &wt, None, None); let after = recover(&before, &wt, TreeId("tree1".to_string())); let transition = DurableTransition::between(&before, &after, &wt) .expect("between should succeed") .expect("strong recovery should produce a durable transition"); - assert_eq!(commit_transition(&db_path, &transition), CasResult::Applied); + assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - assert_round_trip(&db_path, &wt, None, None, &after); + assert_round_trip(db_path, &wt, None, None, &after); - let db = reopen_store(&db_path); + let db = reopen_store(db_path); let store = MutationTraceStore::new(&db); for scope_id in [&scope_a, &scope_b] { let scope_state = store @@ -3432,24 +3477,23 @@ mod tests { .expect("scope row should exist"); assert_eq!(scope_state.status, ScopeStatus::Abandoned); } - - remove_test_db(&db_path); } #[test] fn round_trip_contended_mutation_persists_and_reloads_exactly_after_reopening_the_database() { - let db_path = unique_test_db_path("roundtrip-contended"); + let db_fixture = test_db_path("roundtrip-contended"); + let db_path = db_fixture.path(); let wt = WorktreeId("wt0".to_string()); let scope_a = ScopeId("scope-a".to_string()); let scope_b = ScopeId("scope-b".to_string()); { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); insert_worktree(&db, &wt.0, 0); insert_scope(&db, &scope_a.0, &wt.0, ScopeStatus::Active); insert_scope(&db, &scope_b.0, &wt.0, ScopeStatus::Active); } - let before = load_before_state(&db_path, &wt, None, None); + let before = load_before_state(db_path, &wt, None, None); let attempt = AttemptId("attempt0".to_string()); let prepared = prepare( &before, @@ -3479,9 +3523,9 @@ mod tests { let transition = DurableTransition::between(&before, &after, &wt) .expect("between should succeed") .expect("a contended flush transition should produce a durable transition"); - assert_eq!(commit_transition(&db_path, &transition), CasResult::Applied); + assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - let reloaded = assert_round_trip(&db_path, &wt, None, None, &after); + let reloaded = assert_round_trip(db_path, &wt, None, None, &after); assert_eq!( reloaded.scopes.keys().cloned().collect::>(), BTreeSet::from([scope_a.clone(), scope_b.clone()]), @@ -3493,20 +3537,19 @@ mod tests { Some(ScopeStatus::Active) ); } - - remove_test_db(&db_path); } #[test] fn round_trip_database_failure_changes_only_non_persistent_external_taint() { - let db_path = unique_test_db_path("roundtrip-database-failure"); + let db_fixture = test_db_path("roundtrip-database-failure"); + let db_path = db_fixture.path(); let wt = WorktreeId("wt0".to_string()); { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); insert_worktree(&db, &wt.0, 0); } - let before = load_before_state(&db_path, &wt, None, None); + let before = load_before_state(db_path, &wt, None, None); let after = database_failure(&before, &wt); assert!(!before.external_taint.contains(&wt)); @@ -3526,15 +3569,14 @@ mod tests { DurableTransition should exist to commit" ); - let reloaded = assert_round_trip(&db_path, &wt, None, None, &after); + let reloaded = assert_round_trip(db_path, &wt, None, None, &after); assert!(reloaded.into_protocol_state().external_taint.is_empty()); - - remove_test_db(&db_path); } #[test] fn round_trip_a_replayed_event_key_is_rejected_and_does_not_advance_the_worktree_again() { - let db_path = unique_test_db_path("roundtrip-replay"); + let db_fixture = test_db_path("roundtrip-replay"); + let db_path = db_fixture.path(); let wt = WorktreeId("wt0".to_string()); let scope_id = ScopeId("scope0".to_string()); let event_key = EventKey { @@ -3542,12 +3584,12 @@ mod tests { event_id: EventId("event0".to_string()), }; { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); insert_worktree(&db, &wt.0, 0); insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::Active); } - let before = load_before_state(&db_path, &wt, Some(&scope_id), None); + let before = load_before_state(db_path, &wt, Some(&scope_id), None); let attempt = AttemptId("attempt0".to_string()); let prepared = prepare( &before, @@ -3568,9 +3610,9 @@ mod tests { let transition = DurableTransition::between(&before, &after, &wt) .expect("between should succeed") .expect("the first delivery should produce a durable transition"); - assert_eq!(commit_transition(&db_path, &transition), CasResult::Applied); + assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - let before_replay = load_before_state(&db_path, &wt, Some(&scope_id), Some(&event_key)); + let before_replay = load_before_state(db_path, &wt, Some(&scope_id), Some(&event_key)); assert!(before_replay.processed_events.contains(&event_key)); let replay_attempt = AttemptId("attempt1".to_string()); @@ -3597,8 +3639,385 @@ mod tests { "a rejected replay must produce no durable transition to persist" ); - assert_round_trip(&db_path, &wt, Some(&scope_id), Some(&event_key), &after); + assert_round_trip(db_path, &wt, Some(&scope_id), Some(&event_key), &after); + } + + fn insert_worktree_with_cursor( + db: &RepositoryAgentTraceDb, + worktree_id: &str, + revision: u64, + cursor_tree: &str, + ) { + db.execute( + "INSERT INTO mutation_trace_worktrees + (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) + VALUES (?1, ?2, ?3, 0, 'healthy', 0)", + ( + worktree_id, + cursor_tree, + encode_revision(revision).as_slice(), + ), + ) + .expect("worktree insert should succeed"); + } + + fn insert_event_trees( + db: &RepositoryAgentTraceDb, + worktree_id: &str, + revision: u64, + before_tree: &str, + after_tree: &str, + ) { + db.execute( + "INSERT INTO mutation_trace_events + (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, + attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) + VALUES (?1, ?2, ?3, ?4, 0, 'healthy', 'ineligible_unscoped', NULL, 'flush', NULL, NULL)", + ( + worktree_id, + encode_revision(revision).as_slice(), + before_tree, + after_tree, + ), + ) + .expect("mutation event insert should succeed"); + } + + fn tree_set(trees: [&str; N]) -> BTreeSet { + trees.into_iter().map(|t| TreeId(t.to_string())).collect() + } + + #[test] + fn load_tree_roots_returns_cursor_and_every_event_tree_deduplicated() { + let db_fixture = test_db_path("tree-roots-cursor-and-events"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); + let store = MutationTraceStore::new(&db); + + insert_worktree_with_cursor(&db, "wt-1", 2, "tree-2"); + insert_event_trees(&db, "wt-1", 1, "tree-0", "tree-1"); + insert_event_trees(&db, "wt-1", 2, "tree-1", "tree-2"); + + let roots = store + .load_tree_roots(&WorktreeId("wt-1".to_string())) + .expect("load_tree_roots should succeed"); + + assert_eq!(roots, tree_set(["tree-0", "tree-1", "tree-2"])); + } + + #[test] + fn load_tree_roots_excludes_other_worktrees_trees() { + let db_fixture = test_db_path("tree-roots-excludes-other-worktree"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); + let store = MutationTraceStore::new(&db); + + insert_worktree_with_cursor(&db, "wt-1", 1, "tree-1a"); + insert_event_trees(&db, "wt-1", 1, "tree-0a", "tree-1a"); + + insert_worktree_with_cursor(&db, "wt-2", 1, "tree-1b"); + insert_event_trees(&db, "wt-2", 1, "tree-0b", "tree-1b"); + + let roots = store + .load_tree_roots(&WorktreeId("wt-1".to_string())) + .expect("load_tree_roots should succeed"); + + assert_eq!(roots, tree_set(["tree-0a", "tree-1a"])); + } + + #[test] + fn load_tree_roots_is_empty_for_an_unmaterialized_worktree() { + let db_fixture = test_db_path("tree-roots-unmaterialized-worktree"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); + let store = MutationTraceStore::new(&db); + + insert_worktree_with_cursor(&db, "wt-other", 0, "tree-other"); + + let roots = store + .load_tree_roots(&WorktreeId("wt-missing".to_string())) + .expect("load_tree_roots should return Ok for a worktree with no durable row"); + + assert!(roots.is_empty()); + } + + #[test] + fn load_tree_roots_remains_worktree_scoped() { + let db_fixture = test_db_path("tree-roots-worktree-scoped"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); + let store = MutationTraceStore::new(&db); + + insert_worktree_with_cursor(&db, "wt-1", 1, "tree-1a"); + insert_event_trees(&db, "wt-1", 1, "tree-0a", "tree-1a"); + insert_worktree_with_cursor(&db, "wt-2", 2, "tree-2b"); + insert_event_trees(&db, "wt-2", 1, "tree-0b", "tree-1b"); + insert_event_trees(&db, "wt-2", 2, "tree-1b", "tree-2b"); + + assert_eq!( + store + .load_tree_roots(&WorktreeId("wt-1".to_string())) + .expect("load_tree_roots should succeed"), + tree_set(["tree-0a", "tree-1a"]), + ); + assert_eq!( + store + .load_tree_roots(&WorktreeId("wt-2".to_string())) + .expect("load_tree_roots should succeed"), + tree_set(["tree-0b", "tree-1b", "tree-2b"]), + ); + } + + #[test] + fn load_all_tree_roots_returns_every_worktree_cursor_and_event_tree_deduplicated() { + let db_fixture = test_db_path("all-tree-roots-every-worktree"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); + let store = MutationTraceStore::new(&db); - remove_test_db(&db_path); + insert_worktree_with_cursor(&db, "wt-1", 1, "tree-1a"); + insert_event_trees(&db, "wt-1", 1, "tree-0a", "tree-1a"); + insert_worktree_with_cursor(&db, "wt-2", 2, "tree-2b"); + insert_event_trees(&db, "wt-2", 1, "tree-0b", "tree-1b"); + insert_event_trees(&db, "wt-2", 2, "tree-1b", "tree-2b"); + + let roots = store + .load_all_tree_roots() + .expect("load_all_tree_roots should succeed"); + + assert_eq!( + roots, + tree_set(["tree-0a", "tree-1a", "tree-0b", "tree-1b", "tree-2b",]), + ); + } + + #[test] + fn load_all_tree_roots_deduplicates_a_tree_shared_by_multiple_worktrees() { + let db_fixture = test_db_path("all-tree-roots-shared-tree"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); + let store = MutationTraceStore::new(&db); + + insert_worktree_with_cursor(&db, "wt-1", 1, "tree-shared"); + insert_event_trees(&db, "wt-1", 1, "tree-0a", "tree-shared"); + insert_worktree_with_cursor(&db, "wt-2", 1, "tree-1b"); + insert_event_trees(&db, "wt-2", 1, "tree-shared", "tree-1b"); + + let roots = store + .load_all_tree_roots() + .expect("load_all_tree_roots should succeed"); + + assert_eq!(roots, tree_set(["tree-0a", "tree-shared", "tree-1b"])); + } + + #[test] + fn load_all_tree_roots_is_empty_for_an_empty_repository() { + let db_fixture = test_db_path("all-tree-roots-empty-repository"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); + let store = MutationTraceStore::new(&db); + + let roots = store + .load_all_tree_roots() + .expect("load_all_tree_roots should return Ok for an empty repository"); + + assert!(roots.is_empty()); + } + + fn apply_atomic_cursor_advance(db: &RepositoryAgentTraceDb) { + let guard = TransactionStatement::new( + "UPDATE mutation_trace_worktrees SET cursor_tree = ?1, revision = ?2 + WHERE worktree_id = ?3 AND revision = ?4", + ( + "tree-x", + encode_revision(1).as_slice(), + "wt-b", + encode_revision(0).as_slice(), + ), + ) + .expect("guard statement should build"); + let statements = [TransactionStatement::new( + "INSERT INTO mutation_trace_events + (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, + attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) + VALUES (?1, ?2, ?3, ?4, 0, 'healthy', 'ineligible_unscoped', NULL, 'flush', NULL, NULL)", + ( + "wt-b", + encode_revision(1).as_slice(), + "tree-t", + "tree-x", + ), + ) + .expect("event statement should build")]; + + let applied = db + .execute_transactional_cas_batch( + "atomic cursor advance test", + "retry the operation", + &guard, + &statements, + ) + .expect("the atomic cursor advance should commit"); + assert!(applied, "the CAS guard should have matched revision 0"); + } + + fn select_trees(db: &RepositoryAgentTraceDb, sql: &str) -> BTreeSet { + db.query_map(sql, (), |row| { + let tree: String = row.get(0).context("failed to read a tree column")?; + Ok(TreeId(tree)) + }) + .expect("tree column select should succeed") + .into_iter() + .collect() + } + + /// State-transition coverage only: before the advance `T` is a root + /// through `cursor_tree`; after it, `T` is a root through `before_tree`. + /// This does NOT prove single-statement snapshot isolation — a torn + /// multi-read implementation would still pass this pre/post check. + /// `load_all_tree_roots_reads_every_durable_root_in_one_sql_statement` is + /// the deterministic regression for that property. + #[test] + fn load_all_tree_roots_retains_previous_cursor_after_atomic_cursor_advance() { + let db_fixture = test_db_path("all-tree-roots-retains-previous-cursor"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); + let store = MutationTraceStore::new(&db); + + insert_worktree_with_cursor(&db, "wt-b", 0, "tree-t"); + + let pre = store + .load_all_tree_roots() + .expect("load_all_tree_roots should succeed"); + assert!( + pre.contains(&TreeId("tree-t".to_string())), + "T is a durable root before the transition (via cursor_tree)" + ); + + apply_atomic_cursor_advance(&db); + + let post = store + .load_all_tree_roots() + .expect("load_all_tree_roots should succeed"); + assert!( + post.contains(&TreeId("tree-t".to_string())), + "T is still a durable root after the advance (via before_tree)" + ); + assert!( + post.contains(&TreeId("tree-x".to_string())), + "X becomes a durable root after the advance" + ); + } + + /// Deterministic regression for the actual concurrency boundary: one + /// `load_all_tree_roots()` invocation must read `cursor_tree`, + /// `before_tree`, and `after_tree` through a SINGLE SQL statement / one + /// database snapshot. If it is reimplemented as two or more independent + /// `SELECT`s unioned in Rust, an atomic `cursor T -> X` + + /// `MutationEvent { before = T, after = X }` commit interleaved between + /// those reads produces a torn set that omits `T`. The test constructs + /// that torn set explicitly (an events read, the atomic advance, then a + /// worktrees read, unioned in Rust — losing `T`) and then asserts the + /// production path issues exactly one read statement, so it can never + /// enter the interleaving and always retains `T`. + #[test] + fn load_all_tree_roots_reads_every_durable_root_in_one_sql_statement() { + let db_fixture = test_db_path("all-tree-roots-single-statement"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); + let store = MutationTraceStore::new(&db); + + insert_worktree_with_cursor(&db, "wt-b", 0, "tree-t"); + + let events_first = select_trees( + &db, + "SELECT before_tree AS tree FROM mutation_trace_events + UNION + SELECT after_tree AS tree FROM mutation_trace_events", + ); + assert!( + events_first.is_empty(), + "no event references T before the advance" + ); + + apply_atomic_cursor_advance(&db); + + let cursors_second = select_trees( + &db, + "SELECT cursor_tree AS tree FROM mutation_trace_worktrees", + ); + let torn: BTreeSet = events_first.union(&cursors_second).cloned().collect(); + assert!( + !torn.contains(&TreeId("tree-t".to_string())), + "a two-read implementation loses T across the atomic advance" + ); + + let (roots, statements_issued) = crate::services::db::count_read_statements(|| { + store + .load_all_tree_roots() + .expect("load_all_tree_roots should succeed") + }); + assert_eq!( + statements_issued, 1, + "load_all_tree_roots must read every durable-root column in one SQL statement" + ); + assert!( + roots.contains(&TreeId("tree-t".to_string())), + "the single-statement snapshot always retains T (via before_tree)" + ); + assert!(roots.contains(&TreeId("tree-x".to_string()))); + } + + /// The same single-statement / single-snapshot property, worktree-scoped: + /// one `load_tree_roots(W)` call reads W's `cursor_tree` / `before_tree` / + /// `after_tree` through one statement. A two-read reimplementation + /// (events-for-W, then cursor-for-W) would tear across an atomic cursor + /// advance in exactly the same way. + #[test] + fn load_tree_roots_reads_every_durable_root_in_one_sql_statement() { + let db_fixture = test_db_path("tree-roots-single-statement"); + let db_path = db_fixture.path(); + let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); + let store = MutationTraceStore::new(&db); + + insert_worktree_with_cursor(&db, "wt-b", 0, "tree-t"); + + let events_first = select_trees( + &db, + "SELECT before_tree AS tree FROM mutation_trace_events WHERE worktree_id = 'wt-b' + UNION + SELECT after_tree AS tree FROM mutation_trace_events WHERE worktree_id = 'wt-b'", + ); + assert!( + events_first.is_empty(), + "no event references T before the advance" + ); + + apply_atomic_cursor_advance(&db); + + let cursors_second = select_trees( + &db, + "SELECT cursor_tree AS tree FROM mutation_trace_worktrees WHERE worktree_id = 'wt-b'", + ); + let torn: BTreeSet = events_first.union(&cursors_second).cloned().collect(); + assert!( + !torn.contains(&TreeId("tree-t".to_string())), + "a two-read implementation loses T across the atomic advance" + ); + + let (roots, statements_issued) = crate::services::db::count_read_statements(|| { + store + .load_tree_roots(&WorktreeId("wt-b".to_string())) + .expect("load_tree_roots should succeed") + }); + assert_eq!( + statements_issued, 1, + "load_tree_roots must read every durable-root column in one SQL statement" + ); + assert!( + roots.contains(&TreeId("tree-t".to_string())), + "the single-statement snapshot always retains T (via before_tree)" + ); + assert!(roots.contains(&TreeId("tree-x".to_string()))); } } diff --git a/cli/src/services/resilience.rs b/cli/src/services/resilience.rs index fb1de8c9..2e0484d9 100644 --- a/cli/src/services/resilience.rs +++ b/cli/src/services/resilience.rs @@ -126,15 +126,18 @@ where for attempt in 1..=policy.max_attempts { let started_at = Instant::now(); let outcome = operation(attempt); - let timed_out = started_at.elapsed() >= policy.timeout(); - match (timed_out, outcome) { - (false, Ok(value)) => return Ok(value), - (true, _) => { - last_error = format!("attempt {attempt} timed out after {}ms", policy.timeout_ms); - } - (false, Err(error)) => { - last_error = error.to_string(); + match outcome { + Ok(value) => return Ok(value), + Err(error) => { + last_error = if started_at.elapsed() >= policy.timeout() { + format!( + "attempt {attempt} exceeded {}ms and failed: {error}", + policy.timeout_ms + ) + } else { + error.to_string() + }; } } @@ -220,18 +223,18 @@ mod tests { } #[test] - fn sync_retry_treats_slow_attempt_as_timeout() { + fn sync_retry_returns_a_slow_success_without_retrying_it() { let policy = RetryPolicy { - max_attempts: 1, + max_attempts: 5, timeout_ms: 5, initial_backoff_ms: 0, max_backoff_ms: 0, }; let mut attempts = 0; - let error = run_with_retry_sync( + let value = run_with_retry_sync( policy, - "sync timeout", + "sync slow success", "try again when the resource is available", |attempt| { attempts = attempt; @@ -239,12 +242,64 @@ mod tests { Ok("late success") }, ) + .expect("a completed success must never be reclassified as a timeout"); + + assert_eq!(value, "late success"); + assert_eq!(attempts, 1, "a slow success must not be retried"); + } + + #[test] + fn sync_retry_annotates_a_slow_failure_with_the_elapsed_bound() { + let policy = RetryPolicy { + max_attempts: 1, + timeout_ms: 5, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }; + + let error = run_with_retry_sync::<(), _>( + policy, + "sync slow failure", + "try again when the resource is available", + |_| { + thread::sleep(Duration::from_millis(20)); + Err(anyhow!("connection refused")) + }, + ) .unwrap_err(); - assert_eq!(attempts, 1); assert!(error .to_string() - .contains("Last error: attempt 1 timed out after 5ms")); + .contains("attempt 1 exceeded 5ms and failed: connection refused")); + } + + #[test] + fn sync_retry_does_not_re_execute_a_committed_write_after_a_slow_first_attempt() { + let policy = RetryPolicy { + max_attempts: 5, + timeout_ms: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }; + let mut inserted_keys: Vec = Vec::new(); + let mut calls = 0; + + let result = run_with_retry_sync(policy, "insert once", "retry later", |_| { + calls += 1; + thread::sleep(Duration::from_millis(5)); + if inserted_keys.contains(&1) { + return Err(anyhow!("UNIQUE constraint failed: keys.id")); + } + inserted_keys.push(1); + Ok(()) + }); + + assert!( + result.is_ok(), + "the committed first attempt must be returned" + ); + assert_eq!(calls, 1, "the non-idempotent write must run exactly once"); + assert_eq!(inserted_keys, vec![1]); } #[test] diff --git a/context/cli/mutation-trace-protocol.md b/context/cli/mutation-trace-protocol.md index 8bd83068..6fe7ec17 100644 --- a/context/cli/mutation-trace-protocol.md +++ b/context/cli/mutation-trace-protocol.md @@ -245,7 +245,11 @@ owns the lock, the external-taint fence, the caller-supplied DB provider, one Git snapshot, and a bounded CAS-retry loop; `store.rs` never remaps an existing `ScopeId`'s `actor_kind`/`worktree_id`; `protocol.rs` assumes referenced scopes already exist and stays free of any Git object, DB row, or CAS transaction -concept. +concept. `runtime::ref_reconciliation` +([`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md)) +is imperative durability maintenance *outside* the verified protocol — it never +advances the cursor, chooses attribution, changes scope state, or creates a +`MutationEvent`, only reclaims SCE-owned snapshot refs that are no durable root. ## Authoritative source diff --git a/context/cli/mutation-trace-ref-reconciliation.md b/context/cli/mutation-trace-ref-reconciliation.md new file mode 100644 index 00000000..db8dc8a3 --- /dev/null +++ b/context/cli/mutation-trace-ref-reconciliation.md @@ -0,0 +1,250 @@ +# Mutation-cursor snapshot-ref reconciliation (`runtime::ref_reconciliation`) + +`cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` is a +conservative, per-worktree maintenance pass that removes orphaned / unreferenced +SCE-owned snapshot pins **within the ref namespace of a checkout identity a +current worktree still owns** while retaining every tree any current or +historical durable mutation-cursor state in the repository still references. +Built by the `mutation-cursor-ref-reconciliation` plan +(`context/plans/mutation-cursor-ref-reconciliation.md`). It is deliberately +**not** a guarantee that SCE snapshot refs never accumulate: a namespace whose +checkout identity no current worktree derives is beyond its reach (see +[Scope: an owned checkout-identity namespace only](#scope-an-owned-checkout-identity-namespace-only)). + +`GitSnapshotService::pin_tree` is **create-only per invocation** — a crash, +failed transition, or other interrupted `coordinate()` path can leave a +`refs/sce/mutation-cursor//` pin with no corresponding +durable root. Reconciliation is the reclamation step for exactly that state. It +is **not** a bound on storage growth: every retained `mutation_trace_events` +row keeps its `before_tree` / `after_tree` as durable roots, so a successful +`A → B → C → D` history keeps all four pins. Bounding historical snapshot +storage needs a separate future retention/compaction lifecycle this module does +not design. + +The design is deliberately asymmetric and this bias governs every rule below: +**keeping an unnecessary ref costs disk; deleting a required ref destroys +durable evidence.** False retention is acceptable; false deletion is not. + +`mod ref_reconciliation;` is private in `runtime/mod.rs` (like `coordinator`); +`reconcile_worktree` is `pub` only within `runtime` — no `pub(crate)` re-export, +no hook / command / `diff_traces` wiring yet (invocation timing is the +harness-wiring PR's). + +## Entry point and identity + +```rust +pub fn reconcile_worktree( + repository_root: &Path, + open_db: impl FnOnce() -> anyhow::Result, +) -> Result +``` + +A one-line delegation to `pub(super) fn reconcile_worktree_inner(.., +on_lock_contention: impl FnOnce())` — the deterministic test seam (mirroring +`coordinate` / `coordinate_inner`), visible only within `runtime`. + +Like `coordinate()`, it never accepts a `WorktreeId`, `TreeId`, or ref name and +never opens the DB itself: identity is +`repository_root → resolve_git_dir → read_checkout_id`, `open_db` is +caller-supplied. `read_checkout_id → Ok(None)` returns +`Ok(SkippedNoCheckoutIdentity)` — an observable skip, **not** a zero-count +report: nothing inventoried, `open_db` never called, no ref touched, lock +released, no identity created or recovered; it makes **no** claim the repository +holds no SCE refs for a prior checkout identity (those are the unowned +namespaces below). `Err` is `ReconcileError::CheckoutIdentity` (a corrupt id, not an absent one). + +## Scope: an owned checkout-identity namespace only + +Reconciliation reclaims orphan / unreferenced pins **only** under +`refs/sce/mutation-cursor//` where `` is a checkout identity a **current +worktree still derives** (`resolve_git_dir → read_checkout_id`). The unsupported +case is identity-based, not path-based: + +```text +refs/sce/mutation-cursor//* — does a current worktree derive ? + yes → active namespace (reclaimed here under the WorktreeLock) + no → unowned namespace (no per-worktree pass can ever reach it) +``` + +High-frequency harness traffic against a worktree with a **stable** checkout +identity — where interrupted `coordinate()` runs leave orphan pins — is exactly +the active-namespace case this pass covers. + +A namespace becomes **unowned** whenever no current worktree can derive its +checkout id. Two lifecycles cause this: + +**Case A — deleted linked worktree.** `git worktree remove W` deletes W's +worktree-specific git dir, so its `/sce/checkout-id` is gone; the +`refs/sce/mutation-cursor//*` it created survive in the shared repository +ref namespace. + +**Case B — checkout-identity metadata loss / recreation.** A **present** +worktree's `/sce/checkout-id` disappears (id `A`). `reconcile_worktree` +returns `SkippedNoCheckoutIdentity` and does nothing. A later +`get_or_create_checkout_id` (e.g. the next `coordinate()`) mints a **new** id +`B`; the worktree now operates as `B` while `refs/.../A/*` are unowned. This is +**not** normal operation: it is a metadata-loss / recreation lifecycle the +maintenance model must handle conservatively — reconciliation never recreates +`A` or adopts its namespace. + +**Harness gate.** A persistent / current-worktree harness whose **checkout +identity stays stable** relies on this pass for active-namespace orphan cleanup +and is storage-cleanup complete for this module's scope. Any lifecycle that can +**retire, replace, lose, or recreate** checkout identities (ephemeral linked +worktrees among them, but not only those) can leave unowned namespaces and needs +the future repository-scoped operation below. This is a scope limit, not a bug, +orthogonal to the "reconciliation ≠ historical retention policy" boundary above. + +### Future work: repository-scoped unowned-namespace reconciliation + +Not implemented here — this module adds **no** repository-global namespace scan +and **no** repository-global ref deletion. Recorded shape: + +```text +enumerate refs/sce/mutation-cursor//* (git for-each-ref on the namespace) + ↓ +active checkout ids (enumerate current worktrees → read each one's checkout-id) + ↓ +unowned ids (namespace present, no current worktree derives it) + ↓ +for each unowned namespace, each pinned tree T: + T ∈ durable_roots(repository) → retain + T ∉ durable_roots(repository) → safe deletion candidate +``` + +It must inherit this module's guarantees: the repository-wide durability +invariant (`delete /T` only if `T ∉ durable_roots(repository)`, read +through `load_all_tree_roots()`) because an unowned namespace may still hold the +only SCE ref protecting historical `mutation_trace_events` trees other tooling +needs; the false-retention-over-false-deletion bias; and a hard prohibition on +the shortcut "checkout id is unowned → delete its whole namespace". It is a +separate PR (a repository-global scan and cross-worktree active-id inventory this +module omits), gated behind the same harness-wiring work as invocation timing. + +## Two invariants + +Conflating them — deciding deletion from the target worktree's roots alone — is +the cross-worktree safety bug this design avoids: linked worktrees share one +object database, so an `A`-owned ref can be the last SCE ref protecting a tree only `B` durably requires. + +```mermaid +flowchart TD + inv["list_pins(W) — actual pins under refs/sce/mutation-cursor/<W>/"] + local["load_tree_roots(W)\n(this worktree's cursor + event trees)"] + repo["load_all_tree_roots()\n(every worktree's cursor + event trees)"] + inv --> lc + local --> lc{"local consistency:\ndurable_roots(W) ⊆ pinned_trees(W)?"} + lc -- "no" --> fail["ReconcileError::MissingRequiredPins\n— fail closed, delete nothing"] + lc -- "yes" --> ds + inv --> ds{"deletion safety:\npin.tree ∉ durable_roots(repository)?"} + repo --> ds + ds -- "stale" --> del["delete_pins(stale) — one atomic\ngit update-ref --no-deref --stdin"] + ds -- "retained" --> keep["keep the pin"] +``` + +- **Local consistency** (`load_tree_roots(W)`) is strictly per-worktree. A + missing pin in some *other* worktree never makes `W`'s pass fail — that would + let one worktree's degradation block maintenance everywhere. +- **Deletion safety** (`load_all_tree_roots()`) is repository-wide. `W/T` is + deleted only when `T` is in **no** worktree's durable root set. If `B` durably + needs `T` and `A` also has a `T` pin, `A` retains it as accidental backup + reachability for `B`'s degraded state. + +A DB `TreeId` is a *logical* durability requirement, not itself a Git +reachability edge: it obliges reconciliation to keep at least one SCE ref +protecting that tree, and that retained ref supplies physical Git reachability. +Each root-set query is [one SQL statement over one DB +snapshot](mutation-trace-store.md), which keeps a concurrent atomic +`cursor T → X` + `event T → X` commit on another worktree from tearing the +repository-wide read — no repository-global lock is needed. + +## Locking + +Reconciliation holds the **same** `/sce/mutation-cursor.lock` +`WorktreeLock` that `coordinate()` holds across `pin → CAS → return`, acquired +via `worktree_lock::acquire_inner` and bounded by the module-owned +`RECONCILIATION_LOCK_TIMEOUT` (`Duration::from_secs(10)`, matching the +coordinator's private `WORKTREE_LOCK_TIMEOUT` by intent, not a shared constant). +Mutual exclusion on that one file makes the pin → DB-CAS race structurally +impossible: the reconciler's inventory → diff → delete runs wholly before +`coordinate()` takes the lock (nothing pinned yet) or wholly after it releases +it (tree committed → durable root → retained; never committed → true orphan → +deletable). The lock stays per-worktree; only the durable-root *read* is repository-wide. + +## Algorithm and error contract + +Every step runs under the lock, and every fallible step maps to one dedicated +`ReconcileError` variant — there is no `Other` catch-all: + +| Step | Error on failure | +| --- | --- | +| `resolve_git_dir` | `GitDir` | +| `acquire_inner(RECONCILIATION_LOCK_TIMEOUT)` | `Lock(WorktreeLockError)` | +| `read_checkout_id` → `Err` (corrupt id) | `CheckoutIdentity` | +| `open_db()` provider | `AgentTraceDbUnavailable` | +| `GitSnapshotService::new` | `SnapshotService` | +| `list_pins` → `PinInventoryError::Git` | `PinInventory` | +| `list_pins` → `PinInventoryError::MalformedRef` | `MalformedPin { ref_name, reason }`, delete nothing | +| `load_tree_roots` / `load_all_tree_roots` | `DurableRoots` | +| a target-worktree root has no pin | `MissingRequiredPins { missing }`, delete nothing | +| `delete_pins` transaction | `DeleteTransaction`, delete nothing (atomic-or-nothing) | + +`AgentTraceDbUnavailable` here is a **maintenance** error only: reconciliation +never arms `ExternalTaintMarker`, calls `protocol::*`, or writes a +`mutation_trace_*` row, and never becomes +`CoordinateError::AgentTraceDbUnavailable` — no mutation boundary is being +coordinated (contrast [`mutation-trace-external-taint.md`](mutation-trace-external-taint.md)). + +## Outcome and report + +Both entrypoints return `Result` — +`Reconciled(ReconciliationReport)` | `SkippedNoCheckoutIdentity`, the skip an +`Ok` (never `Err`) and distinct from a zero-work `Reconciled(.. { deleted: 0 })`. + +`ReconciliationReport { local_required: usize, retained: usize, deleted: usize }` +— `local_required = load_tree_roots(W).len()`, `deleted` = stale pins removed, +`retained = actual.len() − deleted`. `retained == local_required` is **not** an +invariant (a pin another worktree needs counts toward `retained` only); for +`Reconciled(report)` the only relation is `report.local_required ≤ report.retained`, +and `SkippedNoCheckoutIdentity` carries no report. + +## Model boundary + +Ref reconciliation is imperative durability maintenance **below** the verified +`spec/mutation_cursor.qnt` protocol — it never advances the cursor, chooses +attribution, changes scope state, or creates a `MutationEvent`, so no Quint +change is warranted. It deletes only SCE-owned refs, never Git objects, and runs +no `git gc` / `git prune` / `git reflog expire`; Git reclaims unreachable +objects on its own schedule. + +## Testing boundary + +`ref_reconciliation.rs`'s inline `#[cfg(test)] mod tests` uses a RAII `Fixture` +(`tempfile::TempDir`, real `git init` repo, checkout id via +`get_or_create_checkout_id`, a schema `RepositoryAgentTraceDb` beside the +worktree so it never perturbs a captured tree) with raw-SQL row seeders, +following the filesystem-touching inline-test precedent (`context/patterns.md`). +Coverage: orphan pin deleted (with/without a worktree row); current-cursor pin +retained without a referencing event; historical `before`/`after` pins retained +after the cursor advances; a pin another worktree durably requires retained +(the `retained > local_required` case); `MissingRequiredPins` fail-closed; a +malformed / symbolic ref fail-closed; idempotence; refs deleted without object +reclamation; and the no-checkout-identity skip (`SkippedNoCheckoutIdentity`; +`open_db` never called; the pin ref byte-identical across the skip). + +`runtime/tests.rs` holds the cross-module integration suite (T09), driven +through the public `reconcile_worktree` (and, for setup, `coordinate`) against +real Git and a real `RepositoryAgentTraceDb`: active-worktree orphan deletion; +current-cursor retention with no referencing event; historical `before`/`after` +retention through a real `A → B → C → D` `coordinate()` history; idempotence; +linked-worktree isolation and cross-worktree degraded-tree retention +(byte-identical pins); a missing required pin failing closed despite another +worktree pinning the same tree; a malformed / symbolic ref failing closed; no +protocol/DB/marker write; no object reclamation; the no-checkout-identity skip; +and two deterministic lock-race regressions (no sleeps): +`reconciliation_blocks_on_the_worktree_lock_and_retains_a_pin_that_becomes_durable_under_it` +(the generic `WorktreeLock` happens-before edge) and +`reconciliation_blocks_until_a_real_coordinate_cas_commits_the_pinned_tree` +(the same edge across the real `coordinate()` `pin → store CAS` path, test-only). + +See also: [`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md), [`mutation-trace-snapshot-service.md`](mutation-trace-snapshot-service.md) (`list_pins` / `delete_pins`), [`mutation-trace-store.md`](mutation-trace-store.md) (`load_tree_roots` / `load_all_tree_roots`), [`mutation-trace-protocol.md`](mutation-trace-protocol.md). diff --git a/context/cli/mutation-trace-runtime-coordinator.md b/context/cli/mutation-trace-runtime-coordinator.md index 082ed7c7..c5744dad 100644 --- a/context/cli/mutation-trace-runtime-coordinator.md +++ b/context/cli/mutation-trace-runtime-coordinator.md @@ -12,8 +12,10 @@ same `#[allow(dead_code)]` precedent as the rest of `mutation_trace`. `coordinator::coordinate()` is the public entrypoint, but `runtime/mod.rs` still declares `mod coordinator;` privately, so `coordinate()` is reachable only from within `runtime` itself (its own tests) for now; a `pub(crate)` -re-export is deferred until a harness adapter needs it. Nothing under -`runtime/` is wired into any hook, command, or `diff_traces` insertion yet. +re-export is deferred until a harness adapter needs it. `mod +ref_reconciliation;` and its `reconcile_worktree` entrypoint are private the +same way. Nothing under `runtime/` is wired into any hook, command, or +`diff_traces` insertion yet. `runtime` depends on `protocol`/`store`/`types` and on `services::checkout`, never the reverse — this is a structural module boundary, not merely a @@ -39,42 +41,29 @@ API end to end. Only harness/command wiring remains. file's mere on-disk existence is never treated as ownership — only a successful OS-level `try_lock()` counts, so a leftover lock file with no active OS lock held against it never blocks a fresh acquirer. -- `cli/src/services/mutation_trace/runtime/git_snapshot.rs` — - `GitSnapshotService::new(repository_root: &Path) -> Result` - resolves `git_dir` once via `git rev-parse --absolute-git-dir`, so - `git_dir` is always an absolute path — even when the caller's - `repository_root` is relative, which matters because every Git subprocess - this service spawns runs with `cwd = repository_root` and - `GIT_DIR = git_dir`; a relative `git_dir` would otherwise be resolved by - the child process against its own already-`repository_root`-joined `cwd`, - double-joining the path. `capture_tree(&self) -> Result` snapshots - the current worktree (staged, unstaged, untracked, and deleted state, - respecting `.gitignore`) into the repository's normal, shared Git object - database, never touching the real index or working tree: it reserves a - unique `/sce/tmp/index-` path via an RAII guard (never - pre-creating the file), probes `HEAD` via a dedicated `head_exists` - helper that inspects the Git exit status directly — status `0` means - `HEAD` resolves, status `1` is `--verify --quiet`'s documented "does not - resolve" signal (a genuinely unborn `HEAD`), and every other status - propagates as an error rather than being treated as empty, since HEAD - absence is a normal Git state but a HEAD-probe failure is a snapshot - failure — then runs `git read-tree HEAD` or, on a genuinely unborn `HEAD`, - the explicit `git read-tree --empty` (never a bare/absent index file), - then `git add -A -- .`, then `git write-tree`, all with only - `GIT_DIR`/`GIT_INDEX_FILE` set — no `GIT_OBJECT_DIRECTORY`/ - `GIT_ALTERNATE_OBJECT_DIRECTORIES` override anywhere. `TreeId` is an opaque - string; nothing assumes a fixed length, so a SHA-256 repository needs no - special handling. `pin_tree(&self, worktree_id, tree) -> Result<()>` makes - a tree durable by creating - `refs/sce/mutation-cursor//` via `git update-ref` — - create-only and idempotent for the same `(worktree_id, tree)` pair — which - is what makes a pinned tree survive `git gc --prune=now`/`git prune - --expire=now`, unlike an unpinned, unreachable tree in the same repository. - `diff_trees(&self, before, after) -> Result` runs `git diff - --binary --full-index --no-ext-diff --no-textconv` between two tree SHAs, - returning the raw diff text `patch.rs::parse_patch` already knows how to - parse. `coordinator.rs` is its only caller, via the `SnapshotCapture` trait - below. +- `cli/src/services/mutation_trace/runtime/git_snapshot.rs` — the isolated Git + snapshot and ref-pinning service (`GitSnapshotService`: + `new`/`capture_tree`/`pin_tree`/`diff_trees`, plus the callerless + worktree-scoped `list_pins` pin inventory — + `Result, PinInventoryError>` — and conditional-atomic + `delete_pins` batch deletion). It writes tree/blob objects into the + repository's normal, shared object database and protects durable trees with + create-only, **direct** `refs/sce/mutation-cursor//` + refs; a symbolic ref inside that namespace is malformed and rejected, and + `delete_pins` uses no-dereference semantics so an inventory→delete ref-type + race cannot escape the inventoried namespace. Full contract in + [`mutation-trace-snapshot-service.md`](mutation-trace-snapshot-service.md). +- `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` — the + conservative per-worktree snapshot-ref maintenance pass — `reconcile_worktree` + / `pub(super) reconcile_worktree_inner` return `Result` (`ReconciliationOutcome` = `Reconciled(ReconciliationReport)` + | `SkippedNoCheckoutIdentity`). Under the worktree's `WorktreeLock` it deletes + only pins whose tree is a durable root of **no** worktree, fails closed if any + local root lacks a pin, and writes no `mutation_trace_*` row or taint marker + (only the namespace of a checkout id a current worktree still derives — a + namespace no current worktree owns, via a deleted worktree or checkout-id + metadata loss/recreation, is future repository-scoped work). Full contract in + [`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md). - `cli/src/services/mutation_trace/runtime/coordinator.rs` — the composition point that drives `protocol.rs`/`store.rs`/`git_snapshot.rs` together. Its `SnapshotCapture` trait (`capture(&self) -> Result`, `pin(&self, @@ -101,11 +90,15 @@ API end to end. Only harness/command wiring remains. the internal generic-over-`SnapshotCapture` pipeline, and clear the marker only on a successful outcome. Identity flows `repository_root → git_dir → WorktreeLock → checkout ID → WorktreeId`; the - DB is not on that chain. (`coordinate()` is a one-line delegation to a - private `coordinate_inner(.., open_db, on_lock_contention: impl FnOnce(), - after_recovery: impl FnMut(u32) -> Result<()>)` test seam; production passes a - no-op contention closure and `|_| Ok(())`.) A `WorktreeLock` - acquisition failure surfaces as `CoordinateError::LockAcquisition`; pre-commit + DB is not on that chain. (`coordinate()` is a one-line delegation to the + `pub(super) coordinate_inner(.., on_lock_contention, after_load, after_recovery)` + test seam — reachable from `runtime::tests`, invisible outside `runtime`; + production passes a no-op for all three. `after_load: impl FnMut(u32)` fires + each CAS attempt after `load_worktree` and before the real `store.commit` CAS; + the reconciliation pin→CAS lock-race regression uses it to pause a real + `coordinate()` between `pin` and CAS. No production behavior change.) A + `WorktreeLock` acquisition failure surfaces as + `CoordinateError::LockAcquisition`; pre-commit marker-I/O and DB-provider failures have their own fail-closed variants, and a post-commit `marker.clear()` failure surfaces as `CoordinateError::MarkerClearAfterCommit { source, committed }` — the boundary @@ -142,7 +135,9 @@ marker arming/clearing, snapshot capture, worktree/scope materialization, recovery, and the CAS retry loop): `coordinate()` acquires it before arming the marker and resolving checkout identity, and holds it until the call returns. It is held on every `coordinate()` call, unlike the checkout-identity-creation -lock. +lock. `ref_reconciliation::reconcile_worktree` acquires the **same** lock file +(bounded by its own `RECONCILIATION_LOCK_TIMEOUT`) before it inventories pins, +reads durable roots, or deletes anything. ## Two distinct locks, two distinct invariants @@ -169,7 +164,7 @@ On-disk layout so far: (runtime::git_snapshot writes here directly) -└── refs/sce/mutation-cursor// (runtime::git_snapshot, one ref per pinned tree, create-only) +└── refs/sce/mutation-cursor// (runtime::git_snapshot, create-only per invocation; orphan/unreferenced pins reclaimed by runtime::ref_reconciliation only for a checkout id a current worktree still derives, every pin for a current or historical durable mutation-cursor root retained; a namespace no current worktree owns — deleted worktree or checkout-id metadata loss/recreation — is unreachable, future repository-scoped work) ``` ## Testing boundary @@ -178,22 +173,14 @@ On-disk layout so far: contention (a second acquirer blocks until the first releases), independence across distinct worktree paths, timing out with a distinct matchable error while the lock is still held, and a leftover lock file with no active OS lock -held against it never blocking a fresh acquirer — each test uses a unique +never blocking a fresh acquirer — each test uses a unique `std::env::temp_dir()` path, following the same filesystem-touching -inline-unit-test precedent already used in `cli/src/services/checkout/mod.rs` -and `cli/src/services/mutation_trace/store.rs` (see `context/patterns.md`). +inline-unit-test precedent as `cli/src/services/checkout/mod.rs` and +`cli/src/services/mutation_trace/store.rs` (see `context/patterns.md`). -`GitSnapshotService`'s inline `#[cfg(test)] mod tests` in `git_snapshot.rs` -uses the same precedent, extended to real per-test `git init` repositories: -index/working-tree preservation across staged/unstaged/untracked/deleted -state, `.gitignore` exclusion, unborn-`HEAD` capture with and without files, -an unexpected `HEAD`-probe failure (a corrupted/missing `.git/HEAD`) -propagating as an error rather than a false empty-baseline capture, a -relative `repository_root` still resolving `git_dir` absolute, survival -after the temp index file is gone, `git gc --prune=now`/`git prune ---expire=now` survival for a pinned tree versus reclamation of a distinct -unpinned tree in the same repository, `pin_tree` idempotency, and -`diff_trees` output shape. +`GitSnapshotService`'s inline `#[cfg(test)] mod tests` in `git_snapshot.rs` uses +the same precedent, extended to real per-test `git init` repositories; coverage +in [`mutation-trace-snapshot-service.md`](mutation-trace-snapshot-service.md). `coordinator.rs`'s inline `#[cfg(test)] mod tests` exercises the internal pipeline against a real temp-file `RepositoryAgentTraceDb`, using a fake, @@ -212,37 +199,33 @@ no write when no worktree row exists yet, and still finds and taints a worktree another caller materializes concurrently during this invocation's own failing capture. Further tests drive the public `coordinate()` against real repositories: the critical-section serialization (a worker's -`coordinate_inner(.., open_db, on_lock_contention)` observes the real -`TryLockError::WouldBlock` branch while a first `WorktreeLock` is held, then -acquires and returns `Ok` once it drops); and the external-taint fence — a -successful call clears the marker, while a snapshot failure, a non-snapshot -failure, a DB-provider `Err`, and an un-armable marker each leave it present -(the last failing closed before the DB provider runs). A further test drives the -private `after_recovery` seam to inject a failure at the exact -recovery-committed / boundary-not-yet-prepared transition and proves the -recovery is durable, the boundary unprocessed with no `MutationEvent`, the -on-disk marker still present, and a later `coordinate()` re-recovering -conservatively off it; `runtime/tests.rs` separately proves an attributable -`Advance` that commits durably then fails its trailing `marker.clear()` surfaces -`MarkerClearAfterCommit` carrying the matching committed outcome (including its -`MutationEvent`). - -`runtime/tests.rs` is `runtime`'s own `#[cfg(test)] mod tests`, holding -cross-module integration tests that drive only the public `coordinate()` API -against real Git repositories (`git init`, `git worktree add`) and real -temp-file `RepositoryAgentTraceDb`s, following the same unique-temp-path -precedent: two linked worktrees of one repository (different `git_dir` → -different lock paths → different `WorktreeId`s) are proven independently -locked by holding one worktree's `WorktreeLock` across a synchronous -`coordinate()` call for the other and observing that call return `Ok` before -the held guard is dropped. Each call is handed a provider closure that opens -the one shared repository-scoped DB path (`coordinate()` never resolves the -DB), and both distinct worktree rows then coexist in it. A first-ever -`agent_trace_storage` resolution and a `coordinate()` call on the same -checkout converge on one checkout identity; and a full failure/recovery -cycle — baseline call, a snapshot-failing call that durably taints the -worktree, then a recovery call that clears the taint before processing its -boundary — runs entirely through the public entrypoint. +`coordinate_inner` observes the real `TryLockError::WouldBlock` branch while a +first `WorktreeLock` is held, then acquires and returns `Ok` once it drops); and +the external-taint fence — a successful call clears the marker, while a snapshot +failure, a non-snapshot failure, a DB-provider `Err`, and an un-armable marker +each leave it present (the last failing closed before the DB provider runs). A +further test drives the private `after_recovery` seam to inject a failure at the +recovery-committed / boundary-not-yet-prepared transition, proving the recovery +durable, the boundary unprocessed with no `MutationEvent`, the marker still +present, and a later `coordinate()` re-recovering off it; another proves an +attributable `Advance` that commits then fails its trailing `marker.clear()` +surfaces `MarkerClearAfterCommit` with the matching committed outcome. The +`after_load` seam is exercised by the reconciliation pin→CAS lock-race regression +([`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md)), pausing a real `coordinate()` between `pin` and CAS. + +`runtime/tests.rs` is `runtime`'s own `#[cfg(test)] mod tests` of cross-module +integration tests against real Git repositories (`git init`, `git worktree +add`) and real temp-file `RepositoryAgentTraceDb`s — the public `coordinate()`, +the public `reconcile_worktree` integration suite (detailed in +[`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md#testing-boundary)), and the `pub(super)` `coordinate_inner` / `reconcile_worktree_inner` lock-race seams. Two linked worktrees (different `git_dir` → +different lock paths → different `WorktreeId`s) are proven independently locked +by holding one worktree's `WorktreeLock` across a synchronous `coordinate()` +call for the other and seeing it return `Ok` only after the guard drops; each +call's provider closure opens the one shared repository-scoped DB path and both +worktree rows coexist in it. A first-ever `agent_trace_storage` resolution and a +`coordinate()` call on one checkout converge on one checkout identity; and a +full baseline → snapshot-failing taint → recovery cycle runs through the public +entrypoint. ## Status @@ -256,7 +239,11 @@ marker is now overlaid onto `database_failure` recovery on the next invocation. wiring remain future work tracked by the `mutation-cursor-external-taint` and `mutation-cursor-runtime-coordinator` plans. -See also: [`mutation-trace-protocol.md`](mutation-trace-protocol.md), +See also: [`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md) +(the per-worktree snapshot-ref maintenance pass under the same `WorktreeLock`), +[`mutation-trace-snapshot-service.md`](mutation-trace-snapshot-service.md) +(the `GitSnapshotService` capture/pin/diff/inventory/delete contract), +[`mutation-trace-protocol.md`](mutation-trace-protocol.md), [`mutation-trace-store.md`](mutation-trace-store.md), [`mutation-trace-external-taint.md`](mutation-trace-external-taint.md) (the `/sce/mutation-cursor-tainted` write-ahead fence armed by diff --git a/context/cli/mutation-trace-snapshot-service.md b/context/cli/mutation-trace-snapshot-service.md new file mode 100644 index 00000000..9abc7feb --- /dev/null +++ b/context/cli/mutation-trace-snapshot-service.md @@ -0,0 +1,148 @@ +# Mutation-trace Git snapshot service (`runtime::git_snapshot`) + +`cli/src/services/mutation_trace/runtime/git_snapshot.rs` — the isolated Git +snapshot and ref-pinning service the +[runtime coordinator](mutation-trace-runtime-coordinator.md) uses to make a +worktree's state durable in the repository's own object database. It writes +tree/blob objects into the repository's normal, shared Git object database and +protects the durable ones with refs under +`refs/sce/mutation-cursor//`, rather than maintaining a +private object store. + +## Construction + +`GitSnapshotService::new(repository_root: &Path) -> Result` +resolves `git_dir` once via `git rev-parse --absolute-git-dir`, so `git_dir` is +always an absolute path — even when the caller's `repository_root` is relative, +which matters because every Git subprocess this service spawns runs with +`cwd = repository_root` and `GIT_DIR = git_dir`; a relative `git_dir` would +otherwise be resolved by the child process against its own +already-`repository_root`-joined `cwd`, double-joining the path. + +## Capture and pin + +`capture_tree(&self) -> Result` snapshots the current worktree (staged, +unstaged, untracked, and deleted state, respecting `.gitignore`) into the +repository's normal, shared Git object database, never touching the real index +or working tree: it reserves a unique `/sce/tmp/index-` path via +an RAII guard (never pre-creating the file), probes `HEAD` via a dedicated +`head_exists` helper that inspects the Git exit status directly — status `0` +means `HEAD` resolves, status `1` is `--verify --quiet`'s documented "does not +resolve" signal (a genuinely unborn `HEAD`), and every other status propagates +as an error rather than being treated as empty, since HEAD absence is a normal +Git state but a HEAD-probe failure is a snapshot failure — then runs +`git read-tree HEAD` or, on a genuinely unborn `HEAD`, the explicit +`git read-tree --empty` (never a bare/absent index file), then `git add -A -- .`, +then `git write-tree`, all with only `GIT_DIR`/`GIT_INDEX_FILE` set — no +`GIT_OBJECT_DIRECTORY`/`GIT_ALTERNATE_OBJECT_DIRECTORIES` override anywhere. +`TreeId` is an opaque string; nothing assumes a fixed length, so a SHA-256 +repository needs no special handling. + +`pin_tree(&self, worktree_id, tree) -> Result<()>` makes a tree durable by +creating `refs/sce/mutation-cursor//` via +`git update-ref` — create-only and idempotent for the same +`(worktree_id, tree)` pair — which is what makes a pinned tree survive +`git gc --prune=now`/`git prune --expire=now`, unlike an unpinned, unreachable +tree in the same repository. + +`diff_trees(&self, before, after) -> Result` runs +`git diff --binary --full-index --no-ext-diff --no-textconv` between two tree +SHAs, returning the raw diff text `patch.rs::parse_patch` already knows how to +parse. + +## Pin inventory and conditional deletion + +**Mutation-cursor pins are direct refs.** Every valid pin is a direct ref +`refs/sce/mutation-cursor//` pointing straight at a tree +object. A **symbolic ref anywhere inside the SCE mutation-cursor namespace is +malformed and rejected** — never followed or normalized — because a symref +under worktree `A`'s prefix could resolve through worktree `B`'s ref, so a pass +reasoning about `A`'s pins would silently be acting on `B`'s. + +`list_pins(&self, worktree_id) -> Result, PinInventoryError>` +inventories the SCE snapshot pins owned by one worktree — `git for-each-ref` +constrained to the single prefix `refs/sce/mutation-cursor//`, +with a NUL-separated `%(refname)%00%(objectname)%00%(objecttype)%00%(symref)` +format. Each ref must satisfy all of: under the exact target prefix; no extra +path segment; `%(symref)` empty (a direct ref); target object type `tree`; +final path component equal to the target SHA. A `git for-each-ref` +execution/exit failure is `PinInventoryError::Git`; any malformed ref inside +the namespace (symbolic ref, non-tree target, name/target SHA mismatch, +unparseable line, extra path segment) is +`PinInventoryError::MalformedRef { ref_name, reason }`, matchable separately. + +`delete_pins(&self, pins: &[PinnedRef]) -> Result<()>` removes exactly the +supplied pins in one atomic **`git update-ref --no-deref --stdin`** +transaction, each `delete` conditioned on the inventoried tree SHA. Two +independent safety properties: + +- **Atomic** — every command commits together at end of input; if any fails + (including a failed old-value check) the whole transaction aborts and no ref + is changed. +- **No dereference** — `--no-deref` makes every `delete` act on the exact ref + name given, never on a ref reached by resolving a symbolic ref. A + direct-ref → symbolic-ref race between inventory and deletion therefore + cannot cause `delete_pins` to touch the symref's target (for example a ref + owned by another worktree). + +Before issuing the transaction, `delete_pins` re-inventories the exact supplied +ref names in one `git for-each-ref` (order-independent) and fails closed unless +every one is still present, still a direct ref to a tree, and still pointing at +the recorded SHA — preferring a clean `Err` over acting on unexpected namespace +state. An empty slice is a successful no-op. + +These are two independent defenses: the preflight catches the common +inventory→delete change (deleted, retargeted, or turned into a symref) before +any `git update-ref` runs; the per-`delete` expected-old-value check inside the +atomic transaction catches a change that lands *after* preflight has passed and +commits nothing. A private test-only `delete_pins_inner(pins, after_preflight)` +seam (production passes a no-op hook) fires between the two so the atomicity +property can be tested with the transaction actually issued. + +`REF_NAMESPACE` (`refs/sce/mutation-cursor`) and the private `pin_ref_prefix` +helper are the single source of truth for the pin path; `pin_tree`, +`list_pins`, and `delete_pins` all derive their ref names from it. + +## Callers + +`coordinator.rs` is the only caller of `capture`/`pin`/`diff_trees` so far, via +its `SnapshotCapture` trait; `list_pins` / `delete_pins` have no caller yet — +the deferred per-worktree ref-reconciliation maintenance pass is their first +consumer. + +## Testing boundary + +`GitSnapshotService`'s inline `#[cfg(test)] mod tests` uses the +filesystem-touching inline-unit-test precedent (see `context/patterns.md`), +extended to real per-test `git init` repositories: index/working-tree +preservation across staged/unstaged/untracked/deleted state, `.gitignore` +exclusion, unborn-`HEAD` capture with and without files, an unexpected +`HEAD`-probe failure (a corrupted/missing `.git/HEAD`) propagating as an error +rather than a false empty-baseline capture, a relative `repository_root` still +resolving `git_dir` absolute, survival after the temp index file is gone, +`git gc --prune=now`/`git prune --expire=now` survival for a pinned tree versus +reclamation of a distinct unpinned tree in the same repository, `pin_tree` +idempotency, `diff_trees` output shape, worktree-scoped `list_pins` inventory +(prefix isolation, a symbolic ref inside the namespace rejected as +`MalformedRef` with `B/T` left intact, other malformed-ref rejection matchable +separately from a `for-each-ref` execution failure), and `delete_pins` +conditional atomic batch deletion (exact removal, empty-slice no-op). Its two +delete defenses are proven separately: + +- **Preflight revalidation** — an inventoried direct ref turned into a symbolic + ref before the call makes `delete_pins` return `Err` and spawn no + `git update-ref` at all, leaving both the symref and its cross-worktree + target untouched. +- **Expected-old-value atomic Git transaction** — proven through the private + test-only `delete_pins_inner(pins, after_preflight)` seam (production + `delete_pins` passes a no-op hook). The test pins two valid trees, passes the + explicit `[valid, mismatched]` batch through preflight, then uses + `after_preflight` to retarget the second ref so + `git update-ref --no-deref --stdin` is genuinely issued; git's per-`delete` + old-value check aborts the batch and the first (valid) ref is left intact. A + sequential conditional-delete implementation in input order would commit the + first delete and fail this. + +See also: [`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md), +[`mutation-trace-protocol.md`](mutation-trace-protocol.md), +[`mutation-trace-store.md`](mutation-trace-store.md). diff --git a/context/cli/mutation-trace-store.md b/context/cli/mutation-trace-store.md index b2527732..ae99bdab 100644 --- a/context/cli/mutation-trace-store.md +++ b/context/cli/mutation-trace-store.md @@ -73,6 +73,48 @@ cold path: it reconstructs one historical `MutationEvent`, including full called from `load_worktree` or from any hook-boundary path, so a projection load never pays for the full historical event set. +## Durable tree-root reads (ref reconciliation) + +`load_tree_roots(worktree)` and `load_all_tree_roots()` are two further +cold-path, read-only queries — siblings of `load_mutation_event`, never +reached from `load_worktree` or a hook-boundary path — that expose the set of +Git tree SHAs the mutation-cursor protocol still durably depends on, for the +per-worktree ref-reconciliation pass (see +[`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md)). + +- `load_tree_roots(worktree) -> BTreeSet` returns one worktree's roots: + its `mutation_trace_worktrees.cursor_tree` plus the `before_tree` and + `after_tree` of every `mutation_trace_events` row for that worktree, + deduplicated. A worktree with no durable row yields the empty set, not an + error. Nothing from another worktree, from `mutation_trace_scopes` / + `mutation_trace_processed_events` / `mutation_trace_event_active_scopes`, or + from transient `AttemptState` / `external_taint` is ever included. +- `load_all_tree_roots() -> BTreeSet` returns the union of those same + three `TreeId` columns across **every** worktree, deduplicated; an empty + repository yields the empty set. This is the reconciler's repository-wide + retention set: linked worktrees share one Git object database, so a ref + owned by one worktree may be the last SCE ref protecting a tree only another + worktree durably requires. + +Each query is backed by a **single SQL statement** — a `UNION` of the +`cursor_tree` / `before_tree` / `after_tree` columns +(`SELECT_TREE_ROOTS_BY_WORKTREE_SQL` / `SELECT_ALL_TREE_ROOTS_SQL`) — run +through one `query_map` call, never independent per-table `SELECT`s unioned in +Rust. So the whole root set is read from one coherent database snapshot: a +concurrent mutation-cursor commit that atomically moves `cursor_tree` from `T` +to `X` and inserts `MutationEvent { before_tree = T, after_tree = X }` in the +same transaction cannot expose a torn set that omits `T` — the statement +observes either the pre-commit snapshot (`cursor_tree` still `T`) or the +post-commit snapshot (`before_tree` is `T`). The one-statement property is the +concurrency boundary here, and it is enforced by a regression test, not left +to code review: a `#[cfg(test)]` read-statement counter in `services::db` +(`count_read_statements`) asserts that one `load_tree_roots` / one +`load_all_tree_roots` call issues exactly one `TursoDb` read — splitting +either into a cursor `SELECT` plus an events `SELECT` fails it. A separate +state-transition test only checks that `T` stays a root across an atomic +cursor advance and is explicitly not treated as proof of snapshot isolation. +These are pure reads: no schema change, no migration, no write path. + ## Write path `MutationTraceStore::commit(transition: &DurableTransition) -> Result` @@ -130,6 +172,8 @@ returns `Err`. caller; retrying with a freshly reloaded revision is the calling adapter's responsibility, not this module's. `runtime::coordinator`'s bounded CAS-retry loop is now that adapter. -- No deletion of terminal (`Closed`/`Abandoned`) scope rows or historical - `mutation_trace_events` rows — scope garbage collection is out of scope for - this plan. +- No row deletion — `store.rs` never deletes a terminal (`Closed`/`Abandoned`) + scope row or a historical `mutation_trace_events` row; scope garbage + collection is out of scope. `load_tree_roots` / `load_all_tree_roots` are + read-only durable-tree queries for ref reconciliation and change nothing + about this. diff --git a/context/context-map.md b/context/context-map.md index f00bb36d..83b691cd 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -26,8 +26,10 @@ Feature/domain context: - `context/cli/mutation-trace-protocol.md` (pure, dependency-free `cli/src/services/mutation_trace/` refinement of the verified `spec/mutation_cursor.qnt` mutation-cursor protocol: the full pure action set — `prepare`/`commit` transition logic, attribution/mutation-event materialization, snapshot-failure/database-failure taint actions, scope abandonment, and recovery with an explicit observed-tree input — plus cross-action sequence/invariant tests and a module-level Quint refinement matrix are all complete (`types.rs`, `protocol.rs`, `mod.rs`, `tests.rs`, registered with `#[allow(dead_code)]`); opaque-newtype identity refinement decisions vs. the Quint model's bounded enums, and the `coordinator.rs` target end-state seam this layout leaves room for but does not create — `store.rs`, `runtime/git_snapshot.rs`, and `runtime/coordinator.rs` (including its public `coordinate()` entrypoint) now exist, built out by the `mutation-cursor-store-persistence` and `mutation-cursor-runtime-coordinator` plans respectively; `protocol.rs`'s pure transitions are not yet wired into any hook or command) - `context/cli/mutation-trace-revision-refinement.md` (the Quint `revision: int` → Rust `WorktreeState::revision: u64` bounded-integer refinement: the private `next_revision` checked-arithmetic helper `commit`/`taint`/`abandon`/`recover` all route through instead of a raw `+ 1`, so a worktree at `revision: u64::MAX` is a guarded no-op/rejection rather than a silent wrap to `0`) - `context/cli/mutation-trace-quint-connect.md` (`#[cfg(test)]`-only Quint Connect model-based-testing harness in `cli/src/services/mutation_trace/mbt/` continuously checking `protocol.rs` against `spec/mutation_cursor.qnt`: the verification-only `mbtAction`/`MbtAction` record-payload transport excluded from comparison, the operation-identity-vs-`MbtStutter` distinction on guarded/no-op branches with its two deterministic regressions, finite ID mapping, the AC5 comparable-state field list, `randomPrepare` staying a single `step` branch, deterministic/generated (500×30, seed-reproducible) test coverage, and the two Nix checks — generic `checks.cli-tests` and dedicated `checks.mutation-trace-quint-connect` — that both require the pinned Quint binary plus the top-level `spec/` directory in `workspaceSrc`'s Nix fileset) -- `context/cli/mutation-trace-store.md` (durable persistence for the mutation-cursor protocol in `cli/src/services/mutation_trace/store.rs`, built by the `mutation-cursor-store-persistence` plan: the one-directional `protocol.rs` -> `DurableTransition::between` (pure structural diff) -> `store.rs` (SQL translation) -> `RepositoryAgentTraceDb` boundary; migration `003_mutation_trace_protocol.sql`'s five tables; `AttemptState`/`external_taint` non-persistence; the 8-byte big-endian `BLOB` revision encoding plus explicit non-`Debug` enum codecs; the bounded hot-path `load_worktree` read vs. the cold-path `load_mutation_event`; `commit`'s single-`BEGIN IMMEDIATE` CAS batch via `TursoDb::execute_transactional_cas_batch`, distinguishing `Conflict`/retryable-transient/deterministic-`Err` outcomes; and the store's non-goals — no Git/filesystem I/O, no attribution/boundary-kind decisions, no retry-after-`Conflict` loop, no terminal-scope garbage collection) -- `context/cli/mutation-trace-runtime-coordinator.md` (imperative-shell runtime layer in `cli/src/services/mutation_trace/runtime/`, built by the `mutation-cursor-runtime-coordinator` plan: the per-worktree OS advisory lock, `runtime::worktree_lock::WorktreeLock::acquire(git_dir, timeout)` at `/sce/mutation-cursor.lock`, bounded `try_lock()` polling with a distinct matchable timeout error and RAII release, and its distinction from the separate checkout-identity-creation lock; and the isolated Git snapshot service, `runtime::git_snapshot::GitSnapshotService` (`capture_tree`/`pin_tree`/`diff_trees`), writing tree/blob objects into the repository's normal object database and pinning durable ones via a create-only, idempotent `refs/sce/mutation-cursor//` ref rather than a private object store; and `runtime::coordinator`'s protocol-integration pipeline (`RuntimeBoundary`/`CoordinateOutcome`/`CoordinateError`, the load → recover-if-needed → prepare/commit CAS-retry loop, and its own bounded snapshot-failure taint-retry loop) plus the public `coordinate(repository_root, boundary, open_db)` entrypoint that acquires the `WorktreeLock`, arms the external-taint write-ahead fence, derives `WorktreeId` from `checkout::get_or_create_checkout_id`, invokes the caller-supplied `open_db` provider so DB acquisition falls inside that fence, drives the pipeline under one held lock, and clears the marker only on success; `runtime/tests.rs` covers the public `coordinate()` API end to end against real linked worktrees and a real Agent Trace DB; an inherited external-taint marker is overlaid onto `protocol::database_failure` recovery on the next invocation, against the single captured snapshot and re-injected across a losing recovery CAS; a `pub(crate)` re-export of `coordinate()` beyond `runtime` and harness/command wiring remain future work) +- `context/cli/mutation-trace-store.md` (durable persistence for the mutation-cursor protocol in `cli/src/services/mutation_trace/store.rs`, built by the `mutation-cursor-store-persistence` plan: the one-directional `protocol.rs` -> `DurableTransition::between` (pure structural diff) -> `store.rs` (SQL translation) -> `RepositoryAgentTraceDb` boundary; migration `003_mutation_trace_protocol.sql`'s five tables; `AttemptState`/`external_taint` non-persistence; the 8-byte big-endian `BLOB` revision encoding plus explicit non-`Debug` enum codecs; the bounded hot-path `load_worktree` read vs. the cold-path `load_mutation_event`; the cold-path read-only `load_tree_roots` (one worktree) / `load_all_tree_roots` (repository-wide) durable-tree-SHA queries for ref reconciliation, each a single-statement `UNION` of `cursor_tree`/`before_tree`/`after_tree` read from one DB snapshot; `commit`'s single-`BEGIN IMMEDIATE` CAS batch via `TursoDb::execute_transactional_cas_batch`, distinguishing `Conflict`/retryable-transient/deterministic-`Err` outcomes; and the store's non-goals — no Git/filesystem I/O, no attribution/boundary-kind decisions, no retry-after-`Conflict` loop, no row deletion) +- `context/cli/mutation-trace-runtime-coordinator.md` (imperative-shell runtime layer in `cli/src/services/mutation_trace/runtime/`, built by the `mutation-cursor-runtime-coordinator` plan: the per-worktree OS advisory lock, `runtime::worktree_lock::WorktreeLock::acquire(git_dir, timeout)` at `/sce/mutation-cursor.lock`, bounded `try_lock()` polling with a distinct matchable timeout error and RAII release, and its distinction from the separate checkout-identity-creation lock; the isolated Git snapshot service `runtime::git_snapshot::GitSnapshotService` (documented in `mutation-trace-snapshot-service.md`); and `runtime::coordinator`'s protocol-integration pipeline (`RuntimeBoundary`/`CoordinateOutcome`/`CoordinateError`, the load → recover-if-needed → prepare/commit CAS-retry loop, and its own bounded snapshot-failure taint-retry loop) plus the public `coordinate(repository_root, boundary, open_db)` entrypoint that acquires the `WorktreeLock`, arms the external-taint write-ahead fence, derives `WorktreeId` from `checkout::get_or_create_checkout_id`, invokes the caller-supplied `open_db` provider so DB acquisition falls inside that fence, drives the pipeline under one held lock, and clears the marker only on success; `runtime/tests.rs` covers the public `coordinate()` API end to end against real linked worktrees and a real Agent Trace DB; an inherited external-taint marker is overlaid onto `protocol::database_failure` recovery on the next invocation, against the single captured snapshot and re-injected across a losing recovery CAS; the private `runtime::ref_reconciliation` per-worktree snapshot-ref maintenance pass (documented in `mutation-trace-ref-reconciliation.md`) shares the same `WorktreeLock`; a `pub(crate)` re-export of `coordinate()` beyond `runtime` and harness/command wiring remain future work) +- `context/cli/mutation-trace-ref-reconciliation.md` (the conservative per-worktree snapshot-ref reconciliation pass in `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs`, built by the `mutation-cursor-ref-reconciliation` plan: `reconcile_worktree(repository_root, open_db)` → `pub(super) reconcile_worktree_inner(.., on_lock_contention)`, both returning `ReconciliationOutcome` (`Reconciled(ReconciliationReport { local_required, retained, deleted })` for a pass that ran | `SkippedNoCheckoutIdentity` — an `Ok`, not an `Err` — when no current checkout identity could be derived), a variant-per-fallible-step `ReconcileError` with no `Other`, and the module-owned `RECONCILIATION_LOCK_TIMEOUT`; the two-invariant model — a strictly per-worktree fail-closed local-consistency check via `load_tree_roots(W)` vs. a repository-wide deletion-safety set via `load_all_tree_roots()` so an `A`-owned ref is retained whenever any worktree still durably needs its tree — run entirely under the same `/sce/mutation-cursor.lock` `WorktreeLock` `coordinate()` holds, with the repository-wide read kept coherent by being one SQL statement / one DB snapshot rather than a repository-global lock; deletes only SCE-owned refs via one atomic `git update-ref --no-deref --stdin`, writes no `mutation_trace_*` row, never arms `ExternalTaintMarker`, runs no `git gc`; imperative durability maintenance below the verified Quint protocol; reclaims orphan/unreferenced refs only for the namespace of a checkout id a current worktree still derives — a namespace no current worktree owns (identity-based: a deleted linked worktree, or checkout-id metadata loss followed by `get_or_create_checkout_id` minting a fresh id on a still-present worktree) is beyond every per-worktree pass and left to a recorded future repository-scoped unowned-namespace operation, so the current pass does not bound all orphan-ref growth; `reconcile_worktree` has no `pub(crate)` re-export and no harness/command wiring yet) +- `context/cli/mutation-trace-snapshot-service.md` (the isolated Git snapshot and ref-pinning service `runtime::git_snapshot::GitSnapshotService` in `cli/src/services/mutation_trace/runtime/git_snapshot.rs`: `new` resolving an absolute `git_dir`, `capture_tree` snapshotting staged/unstaged/untracked/deleted worktree state into the repository's normal object database via a throwaway temp index, `pin_tree` protecting a durable tree with a create-only idempotent **direct** `refs/sce/mutation-cursor//` ref, `diff_trees` emitting `patch.rs`-parseable raw diff text; plus the callerless reconciliation substrate — worktree-scoped `list_pins` inventory returning `Result, PinInventoryError>` that rejects a symbolic ref inside the namespace (mutation-cursor pins are direct refs) as `MalformedRef`, matchable separately from a `git for-each-ref` execution failure, and conditional-atomic `delete_pins` running one `git update-ref --no-deref --stdin` transaction of SHA-conditioned deletes — no-dereference so an inventory→delete direct-ref→symref race cannot escape the inventoried namespace, plus a fail-closed pre-check — that aborts whole if any ref changed since inventory; `REF_NAMESPACE` + `pin_ref_prefix` as the single source of truth for the pin path) - `context/cli/mutation-trace-external-taint.md` (the worktree-local mutation-cursor durability boundary in `cli/src/services/mutation_trace/runtime/external_taint.rs`, built by the `mutation-cursor-external-taint` plan: the `ExternalTaintMarker` primitive — `new(git_dir)`/`exists()`/`persist()`/`clear()` over an empty file at `/sce/mutation-cursor-tainted` whose existence is its entire state, `checkout::persist_checkout_id_inner`-style durability (`sync_data` plus best-effort `#[cfg(unix)]` parent-dir `sync_all`), idempotent persist/clear, `NotFound`-on-clear as success, no `Drop` deletion — as the concrete runtime refinement of the abstract `ProtocolState.external_taint`; armed by the reshaped `coordinate()` entrypoint write-ahead after the `WorktreeLock` and before Agent Trace DB acquisition (a caller-supplied DB provider closure), cleared only on a successful `CoordinateOutcome`, with dedicated fail-closed pre-commit `CoordinateError::ExternalTaintMarker` (`Inspect`/`Persist` only)/`AgentTraceDbUnavailable` variants plus a post-commit `MarkerClearAfterCommit { source, committed }` that carries the durable outcome so a failed trailing clear never hides a committed `MutationEvent`; an inherited marker seeds an invocation-local `external_taint_pending` flag that overlays `protocol::database_failure` onto each freshly loaded projection so `recover` runs once against the captured snapshot, held across a losing recovery CAS and cleared once it lands) - `context/sce/cli-exit-code-contract.md` (stable class-based `sce` process exit-code contract in `cli/src/app.rs`, so automation can branch on failure category without parsing error text) - `context/sce/cli-error-code-taxonomy.md` (stable user-facing `SCE-ERR-*` diagnostic code classes rendered by `cli/src/app.rs`, complementing the numeric exit-code classes) diff --git a/context/overview.md b/context/overview.md index c4f0a36a..1549b10c 100644 --- a/context/overview.md +++ b/context/overview.md @@ -2,7 +2,7 @@ This repository maintains shared assistant configuration for OpenCode, Claude, Pi, and Codex from a single canonical Pkl authoring source. One typed workflow catalog owns the six workflows' shared identity and target routing metadata, while canonical workflow/phase modules own behavior and migrated package-local documents, and target renderers own formatting. Generated target layouts are ephemeral: repository builds consume a pre-Cargo generated payload through `SCE_CLI_GENERATED_INPUT_DIR`, crates.io and Flatpak stage packaging-only fallbacks, and `config/.opencode`, `config/.claude`, `config/.pi`, `config/.agents`, and the generated working-tree SCE config schema are not committed; versioned SCE config schema snapshots live under `schema/v/`. The catalog also marks a workflow `optional` — currently only `brownfield` — which changes nothing about generation and is projected into a generated `config/optional-workflows.json` manifest for install-time consumers. `nix run .#pkl-check-generated` preserves its exact 141-path artifact, metadata/package, phase-reference, internal-reference, optional-workflow-manifest, workflow-orchestration, OpenCode-permission, required-path, and forbidden-path checks while delegating deterministic payload production and inventories to the shared generated-input producer; `nix flake check` runs the same contract. The target matrix contains one manual OpenCode profile plus Claude, Pi, and Codex; the former automated OpenCode profile has been removed. A fourth Pkl renderer, `config/pkl/renderers/codex-content.pkl`, also consumes the same canonical workflow composition to emit skills-only Codex output under `config/.agents/skills/**` (no per-target frontmatter, matching Pi, and no command/prompt layer). Each of the six catalog workflow skills (not `sce-decision`, which has no user-facing entrypoint on any target) additionally carries a Codex-only `agents/openai.yaml`, rendered by `config/pkl/renderers/codex-metadata.pkl` from the shared catalog's `title`/`description` plus an authored `default_prompt`, with `policy.allow_implicit_invocation: false` so these stateful lifecycle workflows activate only from explicit `$sce-` invocation or Codex's `/skills` discovery, never from conversational relevance alone. The Codex renderer also emits a Codex hook registration file (`config/.codex/hooks.json`) and its fail-open install-guidance hook script (`config/.codex/hooks/run-sce-or-show-install-guidance.sh`), both routing every registered lifecycle event through the single command `sce hooks codex`; the generated command resolves the Git root at invocation time and safely invokes the helper from nested cwd or spaced repository paths, failing open silently when Git-root resolution fails. `sce setup --codex` (and `--all`) now installs it as a fourth `SetupTarget`; setup merges the user-owned `.codex/hooks.json` through the shared structural Codex hook-config service, preserving unrelated valid handlers and rejecting malformed or Codex-invalid documents before the atomic swap. `sce doctor` diagnoses that file per required registration (`PresentAndCurrent`/`Missing`/`Stale`, or `Malformed` for the whole document) rather than by whole-document equality, and separately reports whether Codex has actually marked each structurally current registration trusted by reading (never writing) Codex's own `$CODEX_HOME/config.toml` hook-trust state; `sce doctor --fix` repairs structurally unhealthy registrations through the same merge service but never touches trust state (see `context/sce/doctor-human-text-contract.md`). `sce hooks codex` now exists as a typed dispatcher (`cli/src/services/hooks/codex/`): it parses the raw hook JSON into a `CodexHookEvent` and classifies `(hook_event_name, tool_name)` into `UserPromptSubmit`, `Stop`, `PreToolUse(Bash)`, `PostToolUse(apply_patch)`, or a fail-open `NoOp` fallthrough covering every other combination — including `apply_patch` under `PreToolUse`. `UserPromptSubmit` and `Stop` each capture one `messages`/`parts` row (`role="user"`/`role="assistant"`) into the repository Agent Trace DB under the idempotent `cx_` session prefix, `PreToolUse(Bash)` delegates to the existing Bash policy engine, and `PostToolUse(apply_patch)` parses Codex's own `apply_patch` text, resolves paths from the event `cwd` against the real Git root into safe repository-relative paths, normalizes Add/Update evidence into an SCE unified diff under deterministic event-scoped synthetic line identities derived from `tool_use_id`, and persists it as one `diff_traces` row when non-empty (see `context/sce/codex-integration-runtime.md`) — while invalid cwd/path resolution and malformed STDIN payloads fail open with empty stdout, reported model IDs are preserved without fabricated provider prefixes, and missing/blank apply_patch sessions produce no evidence. Delete-File operations and Bash-triggered filesystem mutations remain untracked for Codex. -It also includes a Rust CLI (`sce`) for Shared Context Engineering workflows: auth, config inspection, setup, doctor, Agent Trace hooks and synchronization, bash-policy evaluation, and repository-scoped Agent Trace storage infrastructure. See `context/architecture.md` for module-level boundaries and `context/context-map.md` for the full domain file index. A new pure, dependency-free `cli/src/services/mutation_trace/` module now exists as a Rust refinement of the verified `spec/mutation_cursor.qnt` mutation-cursor protocol; it currently carries domain types and the full pure action set — `prepare`/`commit` transition logic, attribution/mutation-event materialization, snapshot-failure/database-failure taint actions, scope abandonment, and recovery with an explicit observed-tree input — is registered with `#[allow(dead_code)]`. A separate `cli/src/services/mutation_trace/store.rs` persistence layer (built out by the `mutation-cursor-store-persistence` plan) now provides a real, CAS-guarded database call site against the repository-scoped Agent Trace DB, and a `cli/src/services/mutation_trace/runtime/` coordinator layer (built out by the `mutation-cursor-runtime-coordinator` plan) adds the per-worktree advisory lock, the isolated Git snapshot/ref-pinning service, and a public `coordinate()` entrypoint that drives the protocol against a real worktree under that lock; the module is still not wired into any hook or command (see `context/cli/mutation-trace-protocol.md` and `context/cli/mutation-trace-runtime-coordinator.md`). +It also includes a Rust CLI (`sce`) for Shared Context Engineering workflows: auth, config inspection, setup, doctor, Agent Trace hooks and synchronization, bash-policy evaluation, and repository-scoped Agent Trace storage infrastructure. See `context/architecture.md` for module-level boundaries and `context/context-map.md` for the full domain file index. A new pure, dependency-free `cli/src/services/mutation_trace/` module now exists as a Rust refinement of the verified `spec/mutation_cursor.qnt` mutation-cursor protocol; it currently carries domain types and the full pure action set — `prepare`/`commit` transition logic, attribution/mutation-event materialization, snapshot-failure/database-failure taint actions, scope abandonment, and recovery with an explicit observed-tree input — is registered with `#[allow(dead_code)]`. A separate `cli/src/services/mutation_trace/store.rs` persistence layer (built out by the `mutation-cursor-store-persistence` plan) now provides a real, CAS-guarded database call site against the repository-scoped Agent Trace DB, and a `cli/src/services/mutation_trace/runtime/` coordinator layer (built out by the `mutation-cursor-runtime-coordinator` plan) adds the per-worktree advisory lock, the isolated Git snapshot/ref-pinning service, and a public `coordinate()` entrypoint that drives the protocol against a real worktree under that lock, plus a per-worktree `ref_reconciliation` maintenance pass (built out by the `mutation-cursor-ref-reconciliation` plan) that, only for the namespace of a checkout id a current worktree still derives, reclaims orphaned SCE-owned snapshot refs under that same lock while retaining every tree any durable mutation-cursor state still references (a namespace no current worktree owns — via a deleted linked worktree or checkout-id metadata loss/recreation — is out of reach and left to future repository-scoped work); the module is still not wired into any hook or command (see `context/cli/mutation-trace-protocol.md`, `context/cli/mutation-trace-runtime-coordinator.md`, and `context/cli/mutation-trace-ref-reconciliation.md`). The generated `/next-task` workflow persists task-level context-synchronization lifecycle state in each plan (`pending`, `synced`, or `blocked`) so unresolved task synchronization debt survives a session boundary and gates new implementation. Successful `/next-task` execution hands task synchronization an explicit, pre-edit-Git-baseline-relative changed-file list plus implementation, verification, done-check, plan-update, and context-impact evidence, recorded directly on the completed task (`Completed`, `Files changed`, `Result`, `Verify`, `Context impact`, `Context synchronization`); the five-file root context pass remains mandatory. A later-session sync-debt retry reads that same completed task record directly from the plan by plan path and task ID, with no separate persisted synchronization handoff. `/validate` is validation-only: it runs final checks, writes the Validation Report, and reports `validated`, `failed`, or `blocked` without plan-level context synchronization. diff --git a/context/plans/mutation-cursor-ref-reconciliation.md b/context/plans/mutation-cursor-ref-reconciliation.md new file mode 100644 index 00000000..b5b4d23c --- /dev/null +++ b/context/plans/mutation-cursor-ref-reconciliation.md @@ -0,0 +1,3189 @@ +# Plan: mutation-cursor-ref-reconciliation + +## Change summary + +The mutation-cursor runtime pins every captured Git tree under +`refs/sce/mutation-cursor//` in the repository's normal, +shared object database and refs namespace, so a durable snapshot stays +resolvable through `git gc`/`git prune` +(`context/cli/mutation-trace-runtime-coordinator.md`, +`context/plans/mutation-cursor-runtime-coordinator.md`). + +Mutation-cursor snapshot refs are **create-only** during `coordinate()`: +`GitSnapshotService::pin_tree` adds a ref on every invocation and nothing in +the coordinate path ever removes one. A crash, failed transition, or other +interrupted path can therefore leave an SCE-owned pin that has no corresponding +durable mutation-cursor root. + +This plan adds a **conservative per-worktree reconciliation pass** that removes +only such orphaned/unreferenced SCE refs while retaining every tree referenced +by current or historical durable mutation-cursor state. It is best described as +**conservative orphan/unreferenced mutation-cursor snapshot-ref +reconciliation** — not a complete solution to historical mutation-cursor +storage growth. A removed ref is one whose tree is provably outside the +**repository-wide** durable mutation-cursor root set and provably cannot belong +to an in-flight mutation-cursor transition. + +**Reconciliation does not bound storage occupied by retained historical +mutation events.** The current durable-root definition is + +``` +current cursor_tree +∪ every historical MutationEvent.before_tree +∪ every historical MutationEvent.after_tree +``` + +and those historical `mutation_trace_events` rows are retained indefinitely. So +`before_tree` and `after_tree` of every retained historical +`mutation_trace_events` row remain durable roots and therefore remain pinned. A +normal successful history + +``` +events: +A → B +B → C +C → D + +durable roots: +A, B, C, D + +reconciliation: +deletes none +``` + +leaves every one of those pins in place. What reconciliation reclaims is the +complementary case: + +``` +pin X exists +X ∉ durable_roots(repository) + +reconciliation: +delete X +``` + +— a snapshot pin created before a DB/CAS operation that never committed, a +crash artifact, a failed/no-op transition artifact, or another SCE +mutation-cursor pin that is no longer durably referenced. + +This is the deferred step 3 of the runtime completion sequence in +`context/plans/mutation-cursor-runtime-coordinator.md` ("Follow-up PR — Runtime +completion sequence"). Reconciliation is required before high-volume harness +wiring to reclaim orphan/crash snapshot refs produced by interrupted +`coordinate()` executions **in the namespace of a checkout identity a current +worktree still owns** — **not** because it guarantees bounded mutation-history +storage under normal successful usage, and **not** because it reclaims every SCE +ref that can accumulate: a namespace whose checkout identity no current worktree +derives is beyond a per-worktree pass's reach (see "Scope: unowned +checkout-identity namespaces are out of reach" and Design decisions Q16). The +operational consequence is a **harness gate**: a persistent / current-worktree +harness whose checkout identity stays stable can rely on this pass, but any +lifecycle that can retire, replace, lose, or recreate checkout identities is not +storage-cleanup complete until the future repository-scoped unowned-namespace +operation exists. This revision also +makes a missing checkout identity an **observable skipped outcome** +(`ReconciliationOutcome::SkippedNoCheckoutIdentity`) rather than a silent +zero-count report (see "Missing checkout identity is an observable skip" and +Q17), and states precisely what the T04 lock-race regression proves (the +generic `WorktreeLock` happens-before edge, not a production CAS execution) +while adding the **required** exact real-coordinator pin→CAS regression through +the coordinator's existing `after_load` seam (Q18, T07). + +The design is deliberately asymmetric: **keeping an unnecessary ref costs disk +space; deleting a required ref destroys durable evidence.** False retention is +acceptable; false deletion is not. The pass therefore: + +- reads two durable root sets through new bounded, read-only + `MutationTraceStore` queries: the target worktree's own durable tree roots + (`load_tree_roots`), and the union of durable tree roots across **every** + worktree in the repository (`load_all_tree_roots`) — linked worktrees share + one Git object database, so a ref owned by worktree A may be the last SCE + ref protecting a tree that only worktree B durably requires — and it is that + retained ref, not B's database row, that keeps the object reachable to Git. + Each of these APIs executes **exactly one SQL statement** — a `UNION` of the + `cursor_tree`, `before_tree`, and `after_tree` columns — so its complete + logical root set is observed through a single coherent database snapshot, + never assembled in Rust from multiple independent `SELECT`s that a + concurrent mutation-cursor commit could tear across; +- lists only that worktree's pins, validating each against its ref target; +- if any of the **target worktree's own** durable roots has **no** live pin, + fails closed and deletes nothing (the local consistency invariant — a + per-worktree check); +- otherwise deletes exactly the target worktree's pins whose tree is outside + the **repository-wide** durable root set (the deletion safety invariant), + in one atomic `git update-ref --stdin` transaction, each delete conditioned + on the tree SHA observed at inventory time; +- does all of this while holding that worktree's existing + `/sce/mutation-cursor.lock` (`WorktreeLock`), the same lock + `coordinate()` holds across `pin → recovery → prepare → CAS → marker clear + → return`, which is what makes the pin→CAS race structurally impossible. + +This extends the existing `mutation-cursor-runtime-coordinator` and +`mutation-cursor-external-taint` work. It changes no mutation-cursor protocol +semantics, adds no database state, needs no migration, and does not touch the +`ExternalTaintMarker`, `protocol.rs`, `spec/mutation_cursor.qnt`, or the Quint +refinement matrix. It does not run `git gc`/`git prune` — it removes only the +SCE refs Git's own GC already knows how to act on, and lets Git reclaim the +now-unreachable objects on its own schedule. It adds no harness, hook, or +command wiring; `reconcile_worktree()` stays reachable only from within +`runtime`, exactly like `coordinate()`. + +## Storage lifecycle: what this plan bounds and what it does not + +Ref reconciliation is the **reclamation mechanism** once a tree is no longer a +durable root. It is not, by itself, a bound on storage growth for an +indefinitely retained mutation-event history: every retained +`mutation_trace_events` row keeps its `before_tree` and `after_tree` as durable +roots, so their pins are retained for as long as the row is retained. + +Truly bounding historical snapshot storage requires a **separate future +retention/compaction lifecycle** that is explicitly *not* designed or +implemented here: + +``` +MutationEvent(before, after) + ↓ +derive/persist durable diff evidence + ↓ +raw tree snapshots no longer required after retention policy + ↓ +compact/delete historical raw-root references + ↓ +ref reconciliation sees them as unreferenced + ↓ +remove SCE refs + ↓ +Git may reclaim objects on its own GC schedule +``` + +A separate future retention/compaction policy is required to make historical +mutation-event trees stop being durable roots; this plan builds only the +reclamation half of that pipeline. Recorded as future work only — no retention +system is designed or implemented in this plan. + +## Scope: unowned checkout-identity namespaces are out of reach (post-T04 clarification, broadened post-T05/T06 review) + +`reconcile_worktree(repository_root, ..)` derives its owned ref prefix from the +checkout identity a **current worktree still derives** +(`resolve_git_dir → read_checkout_id`). It reclaims orphan / unreferenced pins +**only for a namespace whose checkout identity is still owned**. It is *not* a +guarantee that SCE refs never accumulate. The fundamental unsupported lifecycle +is **identity-based**, not "a worktree directory was deleted": + +``` +refs/sce/mutation-cursor//* + ↓ +does any current worktree derive ? + / \ + yes no + | | + active unowned namespace + namespace (no per-worktree pass can ever inventory or reconcile it) +``` + +A namespace becomes **unowned** in at least two ways: + +**Case A — deleted linked worktree.** `git worktree remove W` deletes W's +worktree-specific git dir, so its `/sce/checkout-id` disappears; the +`refs/sce/mutation-cursor//*` it created live in the shared repository refs +and survive. No surviving worktree can derive that id. + +**Case B — checkout-identity metadata loss / recreation.** A **still-present** +worktree's `/sce/checkout-id` disappears (id `A`); +`get_or_create_checkout_id` mints a fresh UUID when that file is absent, so a +later identity-creating path (the next `coordinate()`, setup, a hook) creates +`B`. The worktree now operates as `B` while `refs/sce/mutation-cursor/A/*` +remain — unowned, though the physical worktree still exists. Between the loss +and the recreation, `reconcile_worktree` returns `SkippedNoCheckoutIdentity` +and does nothing. This is **not** normal operation — it is a metadata-loss / +recreation lifecycle the maintenance model must handle conservatively; +reconciliation never recreates `A` or adopts its namespace. + +``` +/sce/checkout-id per-worktree, easily lost +refs/sce/mutation-cursor// repository-shared, survives + + A: git worktree remove → checkout-id file gone with the worktree + B: checkout-id file deleted → skip → get_or_create mints a fresh id + either way → refs/...//* unreachable by every per-worktree pass +``` + +The current pass is the right mechanism for its stated job — **high-frequency +harness traffic against a worktree with a stable checkout identity**, where +interrupted `coordinate()` runs leave orphan pins reclaimed under the same +`WorktreeLock`. Any lifecycle that can **retire, replace, lose, or recreate** +checkout identities leaves whole unowned namespaces the per-worktree reconciler +structurally cannot reach. That is a **separate repository-scoped lifecycle**, +recorded as future work (see Design decisions Q16), not implemented here: + +``` +enumerate refs/sce/mutation-cursor//* (git for-each-ref on the namespace) + ↓ +enumerate current worktrees → read each worktree's checkout-id → active checkout ids + ↓ +namespace id ∉ active checkout ids → unowned / retired checkout namespace + ↓ +for each unowned namespace, each pinned tree T: + T ∈ durable_roots(repository) → retain + T ∉ durable_roots(repository) → safe candidate for deletion +``` + +The same repository-wide durability invariant still governs it — +`delete /T only if T ∉ durable_roots(repository)` — because an +unowned namespace may still own the only SCE ref protecting historical +`mutation_trace_events` trees other tooling will need. It must keep preferring +**false retention over false deletion**; the unsafe shortcut "checkout id is +unowned → delete its whole namespace" is explicitly forbidden. + +### Operational consequence: the harness gate + +``` +persistent / current worktree, checkout identity stays stable + ↓ +per-worktree reconciliation reclaims orphan/unreferenced refs in that +namespace → storage cleanup is complete for this plan's scope + +any lifecycle that can retire / replace / lose / recreate a checkout identity +(create → run agent → delete a linked worktree; a lost checkout-id file; …) + ↓ +old namespace becomes unowned; its refs remain; no per-worktree pass can reach +them → storage cleanup is NOT complete until the repository-scoped +unowned-namespace reconciliation (Q16) exists +``` + +So this does **not** block ordinary persistent-worktree harness wiring — a +persistent / current-worktree harness **whose checkout identity remains stable** +can proceed on the current pass alone. But **any harness lifecycle that can +retire, replace, lose, or recreate checkout identities MUST NOT be treated as +storage-cleanup complete** until the repository-scoped operation exists. This +gate is recorded in Q15 (invocation timing) and Q16 (the future operation). + +## Missing checkout identity is an observable skip, not a silent zero (post-T03 clarification) + +On the `read_checkout_id → Ok(None)` branch (no current checkout identity to +derive an owned ref prefix from), T03 returned +`Ok(ReconciliationReport { 0, 0, 0 })` — indistinguishable from a real pass +that examined the namespace and found nothing stale. Those two states have +different operational meaning, so T06 makes the skip **explicit** as a +distinct, non-error outcome — `ReconciliationOutcome::SkippedNoCheckoutIdentity` +(Design decisions Q17). A real zero-work pass instead returns +`Ok(Reconciled(ReconciliationReport { 0, 0, 0 }))` (Q6). The skip means only: +*no owned namespace was inventoried, no DB provider was called, no durable-root +comparison was performed, no ref was deleted, no identity was created or +recovered.* It makes **no** claim that the repository holds no SCE +mutation-cursor refs for a prior checkout identity — those may be exactly the +unowned namespaces above (a `SkippedNoCheckoutIdentity` return is itself Case B +of that lifecycle in progress: the metadata is gone and a fresh identity has +not yet been minted). + +## Core invariants + +Two separate invariants govern the pass. Conflating them — deciding deletion +from the target worktree's roots alone — is the cross-worktree safety bug this +design exists to avoid. + +### Local consistency invariant + +For the target worktree `W`: + +``` +durable_roots(W) ⊆ pinned_trees(W) +``` + +If this is false — some tree `W`'s own durable evidence references has no live +pin — reconciliation **fails closed and deletes nothing** +(`ReconcileError::MissingRequiredPins`). This check is strictly per-worktree: +a missing pin in some *other* worktree `B` never makes `A`'s pass fail. + +### Deletion safety invariant + +A ref `refs/sce/mutation-cursor//T` owned by `W` may be deleted only if `T` +is absent from the durable roots of **every** worktree in the repository: + +``` +delete W/T ⟺ T ∉ durable_roots(repository) + = T ∉ ⋃_{V ∈ worktrees} durable_roots(V) +``` + +A durable DB `TreeId` does not itself make a Git object reachable. It is a +**logical durability requirement**: it tells reconciliation that at least one +SCE Git ref protecting that tree must be retained, and that retained Git ref +is what supplies **physical Git reachability** to Git GC. Linked worktrees +share one Git object database, so an `A`-owned ref can be the last SCE ref +protecting a tree that only worktree `B` durably requires +(`B`'s `cursor_tree`, or a `before_tree` / `after_tree` of one of `B`'s +historical events). Deciding staleness as `actual_A − durable_roots(A)` alone +would let `A`'s pass delete that last ref, leaving nothing to protect `B`'s +durable cursor/evidence tree from a later `git gc`. The **retention set** is +therefore repository-wide; the +**lock** stays per-worktree (Q1, Q2). If `B` requires `T` and `A` also has a +`T` pin, `A` retains it — `A`'s otherwise-stale ref then acts as conservative +accidental backup reachability for `B`'s degraded state. + +### Historical retention is deliberate + +A historical event tree — the `before_tree` or `after_tree` of any retained +`mutation_trace_events` row — is a member of `durable_roots(repository)`: + +``` +historical event tree ∈ durable_roots(repository) + ↓ +must be retained +``` + +Reconciliation therefore deliberately prefers historical retention. It never +deletes a pin for a tree that any current or historical durable mutation-cursor +state still references, and it does not attempt to bound the storage those +retained historical trees occupy — that needs the separate future +retention/compaction lifecycle described above. AC2 (unreferenced pin → +delete) and AC4 (historically referenced pin → retain) together fix this +contract. + +## Acceptance criteria + +How this plan is proven complete. Each criterion is observable and names the +check that proves it. `/validate` runs these checks; no task in the stack +performs final validation. All `cargo` invocations go through +`./scripts/run-cli-cargo.sh` (per `context/patterns.md`); test paths use the +crate module path `services::mutation_trace::runtime::…` / +`services::mutation_trace::store::…`. + +- [x] AC1: The store layer exposes two durable-root reads and each is exact. + `load_tree_roots(W)` returns exactly the union of `W`'s + `mutation_trace_worktrees.cursor_tree` and every + `mutation_trace_events.before_tree` / `mutation_trace_events.after_tree` row + for `W`, deduplicated — and nothing from any other worktree, any other + table, or `AttemptState`/`external_taint`; a worktree with no durable row + yields the empty set (not an error). `load_all_tree_roots()` returns the + union of those same three `TreeId` columns across **every** worktree in the + repository, deduplicated; an empty repository yields the empty set (not an + error). A tree that two worktrees both reference appears once in + `load_all_tree_roots()`. + Each root-set API executes **exactly one SQL statement** covering the + cursor, before-tree, and after-tree roots (a `UNION` of the three columns, + driven through a single `query_map` call), so a concurrent mutation-cursor + commit — which atomically moves `cursor_tree` from `T` to `X` and inserts a + `MutationEvent { before_tree = T, after_tree = X }` in the same transaction — + cannot expose a mixed pre/post-commit root set: the one statement observes + either the pre-commit snapshot (`cursor_tree` still contains `T`) or the + post-commit snapshot (`before_tree` contains `T`), and `T` is retained under + both. There is no snapshot in which `cursor_tree` no longer contains `T` + while `before_tree` does not yet contain it. This must not depend on any + ordering of separate reads ("query cursor first"); it is a structural + property of the single-statement snapshot. A new DB transaction API is + **not** required if one `SELECT`/`UNION` statement already provides these + snapshot semantics. + + Two separable properties, proven by two separate tests, must not be + conflated: + + - **State-transition retention** (`…retains_previous_cursor_after_atomic_cursor_advance`): + before an atomic `cursor T → X` + `event T → X` advance, `T` is a durable + root via `cursor_tree`; after it, `T` is a durable root via `before_tree`. + A pre/post read straddling the advance still sees `T` both times. This is + necessary but **not sufficient** — a torn multi-read implementation would + also pass it, so this test is explicitly *not* evidence of snapshot + isolation. + - **Single-statement snapshot enforcement** + (`…reads_every_durable_root_in_one_sql_statement`, for both + `load_all_tree_roots` and `load_tree_roots`): the deterministic regression + for the actual concurrency boundary. It (a) constructs the torn set + explicitly — an events read, the atomic advance committed between the + reads, then a worktrees read, unioned in Rust, which loses `T` — and (b) + asserts, via the `TursoDb` read-statement counter + (`crate::services::db::count_read_statements`), that one production + `load_*_tree_roots` call issues **exactly one** read statement, so it can + never enter that interleaving and always retains `T`. Reimplementing + either query as two independent `SELECT`s (cursor, then events, or events, + then cursor) makes the counter observe `2` and fails the test. **One SQL + statement is the concurrency boundary** because it is the unit of DB + snapshot isolation: everything the statement reads comes from a single + coherent MVCC snapshot, whereas two statements are two snapshots a + concurrent commit can fall between. + - Validate: `services::mutation_trace::store::tests::load_tree_roots_returns_cursor_and_every_event_tree_deduplicated`, + `services::mutation_trace::store::tests::load_tree_roots_excludes_other_worktrees_trees`, + `services::mutation_trace::store::tests::load_tree_roots_is_empty_for_an_unmaterialized_worktree`, + `services::mutation_trace::store::tests::load_tree_roots_remains_worktree_scoped`, + `services::mutation_trace::store::tests::load_all_tree_roots_returns_every_worktree_cursor_and_event_tree_deduplicated`, + `services::mutation_trace::store::tests::load_all_tree_roots_deduplicates_a_tree_shared_by_multiple_worktrees`, + `services::mutation_trace::store::tests::load_all_tree_roots_is_empty_for_an_empty_repository`, + `services::mutation_trace::store::tests::load_all_tree_roots_retains_previous_cursor_after_atomic_cursor_advance` (state-transition retention only — not proof of snapshot isolation), + `services::mutation_trace::store::tests::load_all_tree_roots_reads_every_durable_root_in_one_sql_statement` and + `services::mutation_trace::store::tests::load_tree_roots_reads_every_durable_root_in_one_sql_statement` (the deterministic single-statement snapshot regression: torn two-read set constructed explicitly, then production path asserted to issue exactly one read statement via `count_read_statements`) +- [x] AC2: An **orphan / unreferenced** pin — a pinned tree that is in no + durable root anywhere in the repository, the observable post-crash / + post-no-op state `pin exists ∧ durable root does not` — is deleted by + reconciliation, whether or not the worktree has a durable row at all; + `git for-each-ref` no longer lists that ref afterward. + This criterion is about an unreferenced pin (`unreferenced pin → delete`), + **not** about a merely old one: a pin whose tree is still a current or + historical durable root is retained (AC3, AC4), no matter how old the ref is. + The reconciler does not care how that orphan state arose, so the tests + construct it directly: capture a tree, pin it via `GitSnapshotService`, + create no durable root for it, then run the pass. + - Validate: `services::mutation_trace::runtime::ref_reconciliation::tests::orphan_pin_with_a_worktree_row_is_deleted`, + `services::mutation_trace::runtime::ref_reconciliation::tests::orphan_pin_with_no_worktree_row_is_deleted`, + and end-to-end `services::mutation_trace::runtime::tests::a_pin_with_no_durable_root_is_reclaimed_by_a_later_reconciliation` +- [x] AC3: A pin whose tree is the worktree's current + `mutation_trace_worktrees.cursor_tree` survives reconciliation even when no + `mutation_trace_events` row references that tree. + - Validate: `services::mutation_trace::runtime::ref_reconciliation::tests::current_cursor_pin_is_retained_without_a_referencing_event` +- [x] AC4: Pins for the `before_tree` and `after_tree` of historical + `mutation_trace_events` rows survive reconciliation after the worktree's + cursor has moved on to a later tree, so a future `diff_trees(before, after)` + over that historical interval stays possible. + A historical event tree is a member of `durable_roots(repository)` and is + therefore retained (`historically referenced pin → retain`). Reconciliation + deliberately prefers historical retention; it does **not** bound the storage + these retained trees occupy — for the history `A → B → C → D` with all three + events retained, reconciliation deletes none of `{A, B, C, D}`. Bounding that + storage is separate future retention/compaction work (see "Storage + lifecycle"). + - Validate: `services::mutation_trace::runtime::ref_reconciliation::tests::historical_event_before_and_after_pins_are_retained_after_the_cursor_advances` +- [x] AC5: Reconciliation cannot inventory or delete a pin while another owner + holds the worktree's `WorktreeLock` and, under that lock, makes a newly + pinned tree durable. A deterministic test holds one worktree's `WorktreeLock`, + starts `ref_reconciliation::reconcile_worktree_inner` (the `pub(super)` seam) + on another thread with a channel-signalling `on_lock_contention` closure and + proves it blocks on that same lock, then — still holding the lock — makes a + pinned tree X a durable root, releases the lock, and asserts the + now-unblocked reconciliation retains X (`deleted == 0`). No sleeps; the proof + is the **`WorktreeLock` happens-before edge**. This is the generic shared-lock + ordering guarantee — it does **not** claim the regression executes the + production coordinator `capture → pin → CAS` path (X is made durable directly + in the test). The exact real-coordinator pin→CAS regression is a separate, + **required** test (T07, AC16); T04 is kept as the generic proof. + - Validate: `services::mutation_trace::runtime::tests::reconciliation_blocks_on_the_worktree_lock_and_retains_a_pin_that_becomes_durable_under_it` +- [x] AC6: Given target worktree W whose local durable root set is `{A, B}`, + W's pins `{A, X}` (root B has no pin), and no other worktree durably + referencing X, reconciliation returns a distinct `missing-required-pins` + error naming B, deletes zero refs, and leaves both A's and X's pins in + place. + - Validate: `services::mutation_trace::runtime::ref_reconciliation::tests::a_missing_required_pin_fails_closed_and_deletes_nothing` +- [x] AC7: A ref inside `refs/sce/mutation-cursor//` that is a + **symbolic ref**, whose target is not a tree object, or whose name suffix + disagrees with its target SHA, or whose `for-each-ref` line does not parse, + makes `list_pins` return `PinInventoryError::MalformedRef { ref_name, reason + }`, which `reconcile_worktree` maps to `ReconcileError::MalformedPin { + ref_name, reason }`; reconciliation then deletes nothing. Mutation-cursor + pins are direct refs; a symbolic ref inside the SCE mutation-cursor namespace + is malformed and rejected, never followed or normalized. + - Validate: `services::mutation_trace::runtime::git_snapshot::tests::list_pins_rejects_a_ref_whose_target_is_not_a_tree`, + `services::mutation_trace::runtime::git_snapshot::tests::list_pins_rejects_a_ref_whose_name_disagrees_with_its_target`, + `services::mutation_trace::runtime::git_snapshot::tests::list_pins_rejects_a_symbolic_ref_inside_the_mutation_cursor_namespace`, + `services::mutation_trace::runtime::ref_reconciliation::tests::a_malformed_namespace_ref_fails_closed_and_deletes_nothing` +- [x] AC8: Running reconciliation twice with no intervening state change: + the first run deletes the stale pins and returns success; the second run + deletes zero and returns success with identical `local_required`/retained + counts. + - Validate: `services::mutation_trace::runtime::ref_reconciliation::tests::reconciliation_is_idempotent` +- [x] AC9: With linked worktrees A and B sharing one object database and ref + namespace, `reconcile_worktree` for A operates only under + `refs/sce/mutation-cursor//`, acquires only A's `WorktreeLock`, leaves + every `refs/sce/mutation-cursor//` ref untouched, requires no pause in + B's coordinating, and does not make unresolvable any object that a B ref + still names, **or that a B durable root still requires SCE to protect with a + retained ref** — even when A and B pinned byte-identical tree content. (A B + durable root is a logical durability requirement, not itself a Git + reachability edge: it obliges reconciliation to keep at least one SCE Git ref + protecting that tree, and that retained ref is what keeps the object + reachable to Git.) This includes the cross-worktree + degraded-state case: B durably references a tree T, B's own + `refs/sce/mutation-cursor//T` is deliberately absent, A owns + `refs/sce/mutation-cursor//T`, and A does not durably reference T — + `reconcile_worktree(A)` must **retain** `refs/sce/mutation-cursor//T` + because T is a repository-wide durable root, and the retained ref — not B's + database row itself — keeps T resolvable via + `git cat-file -t` afterward. This is the canonical proof that reconciliation + cannot convert another worktree's degraded-but-recoverable state into + evidence loss. + - Validate: `services::mutation_trace::runtime::tests::reconcile_one_linked_worktree_leaves_the_other_worktrees_pins_and_shared_objects_intact`, + `services::mutation_trace::runtime::tests::reconcile_a_retains_its_pin_when_another_worktree_durably_requires_the_same_tree` +- [x] AC10: The stale-pin deletion is one atomic + `git update-ref --no-deref --stdin` transaction (no-dereference, so a + `delete` can only ever remove the exact ref name given, never a ref reached + through a symbolic ref) in which every `delete` is conditioned on the tree + SHA recorded at inventory time; if any inventoried ref no longer points at + that SHA when the transaction runs, the whole transaction aborts and **no** + ref is deleted. + + Two independent defenses, proven by two different tests: + + 1. **Preflight revalidation** — before the transaction, `delete_pins` runs a + fail-closed re-inventory that returns `Err` (deleting nothing) if any + supplied ref has been deleted, retargeted, or turned into a symbolic ref + since inventory. This catches the common inventory→delete change and no + `git update-ref` is ever spawned. Proven by an inventoried direct ref + turned into a symbolic ref onto another worktree's ref before + `delete_pins`, asserting `Err` with both the symref and its target + untouched. + 2. **Expected-old-value atomic Git transaction** — for a change that lands + *after* preflight has already passed, the per-`delete` old-value check + inside the single `git update-ref --no-deref --stdin` transaction rejects + the batch and Git commits nothing. Proven by a deterministic test hook + (`delete_pins_inner`'s `after_preflight` seam) that mutates the second of + two already-preflight-passed refs, so the transaction is actually issued + with `[delete valid_ref A, delete mismatched_ref B]`, the second + expected-old-value check fails, and the first (valid) ref is left + untouched. A sequential conditional `git update-ref -d` implementation in + input order would commit the first delete before hitting the stale second + and would fail this test. + + The public `reconcile_worktree` path is only asserted to route its stale + batch through `delete_pins`, not to independently schedule a mid-pass race — + the `after_preflight` seam is private to `git_snapshot.rs` and test-only. + - Validate: `services::mutation_trace::runtime::git_snapshot::tests::delete_pins_atomically_aborts_when_a_ref_changes_after_preflight`, + `services::mutation_trace::runtime::git_snapshot::tests::delete_pins_refuses_to_act_when_an_inventoried_direct_ref_became_a_symbolic_ref` +- [x] AC11: A reconciliation pass performs no mutation-cursor protocol or + durability write. Runtime-observable, asserted by the integration test: after + a pass, `mutation_trace_worktrees` / `mutation_trace_scopes` / + `mutation_trace_events` / `mutation_trace_processed_events` / + `mutation_trace_event_active_scopes` row counts and the target worktree's + `revision` / `tainted` / `failure_kind` / `needs_rebaseline` / `cursor_tree` + are byte-identical to before, and no `/sce/mutation-cursor-tainted` + marker is created. Separately, by **inspection** (not a runtime assertion): + `cli/migrations/agent-trace-repository/` still contains exactly + `001`/`002`/`003` — this plan adds no migration. + - Validate: `services::mutation_trace::runtime::tests::reconciliation_makes_no_protocol_or_marker_write`; + inspection: `ls cli/migrations/agent-trace-repository/` shows only the three existing files +- [x] AC12: A reconciliation pass invokes no `git gc` / `git prune` / `git + reflog expire` and no object-reclaiming command. Set up an object O that is + reachable before the pass **only** through a stale SCE pin + `refs/sce/mutation-cursor//O` (O is in no durable root). Reconciliation + removes that stale ref; immediately afterward O is unreachable, yet + `git cat-file -t` still resolves O — because reconciliation deleted the ref + but ran no `git gc` / `git prune`, so Git has not yet reclaimed the + now-unreachable object. The assertion is only about immediate + post-reconciliation resolvability, before any explicit GC; the plan does not + rely on the object surviving indefinitely. + - Validate: `services::mutation_trace::runtime::ref_reconciliation::tests::reconciliation_deletes_refs_without_reclaiming_objects` +- [x] AC13: The plan and the durable reconciliation context state, in precise + terms, that the fundamental unsupported lifecycle is **identity-based** — an + SCE ref namespace exists and **no current worktree owns / derives that + checkout id** — not merely "a linked worktree was deleted". A deleted linked + worktree stays documented as one concrete way a checkout identity becomes + unowned (Case A); checkout-id metadata loss followed by + `get_or_create_checkout_id` minting a fresh id, leaving the old namespace + unowned while the physical worktree still exists, is documented as the other + (Case B), described as a metadata-loss / recreation lifecycle rather than + normal operation. A future repository-scoped "unowned checkout-identity + reconciliation" operation is recorded (repository-scoped namespace scan → + active checkout ids from current worktrees → unowned ids → + per-unowned-namespace tree-vs-`durable_roots(repository)` comparison → delete + only non-durable trees), with `delete /T only if + T ∉ durable_roots(repository)` preserved and the "checkout id is unowned → + delete the whole namespace" shortcut explicitly forbidden. No repository-global + scan or deletion behavior is implemented in this plan. + - Validate: inspection — the "Scope: unowned checkout-identity namespaces" + section of this plan and Design decisions Q16; + `context/cli/mutation-trace-ref-reconciliation.md` and + `context/cli/mutation-trace-runtime-coordinator.md` carry the same + identity-ownership framing and both Case A / Case B; + `git diff main -- cli/src/services/mutation_trace/` shows + `list_pins` still constrains `git for-each-ref` to the single + `refs/sce/mutation-cursor//` prefix and no new + `git worktree list` / repository-wide `refs/sce/**` enumeration or + repository-global ref deletion was added. +- [x] AC14: `reconcile_worktree` returns a result type that makes a real + zero-work reconciliation (`Reconciled(ReconciliationReport { .., deleted: 0 })`) + distinguishable from a skip because no checkout identity could be derived + (`SkippedNoCheckoutIdentity`), and the skip is **not** an `Err`. The skip is + returned with the `WorktreeLock` already released and no checkout identity + created. + - Validate: `services::mutation_trace::runtime::ref_reconciliation::tests::no_checkout_identity_returns_a_distinct_skipped_outcome` +- [x] AC15: On the missing-checkout-identity path, the caller's `open_db` + provider is never invoked, no pin inventory is attempted, and every existing + ref under the repository's SCE namespace is left **structurally** identical. + A test wiring an `open_db` provider that panics if called proves it is not + called, and captures the pre-seeded pin ref's full `git for-each-ref` + representation — `%(refname)` / `%(objectname)` / `%(objecttype)` / + `%(symref)` — before the skip and asserts it is byte-identical afterward, so a + direct ref silently turning symbolic (or any other structural change that + still resolves to the same SHA) would fail. Read with a test-local `git` + helper, not `list_pins`, which needs the checkout identity the test removes. + - Validate: `services::mutation_trace::runtime::ref_reconciliation::tests::a_missing_checkout_identity_skip_touches_no_db_and_no_ref` +- [x] AC16: The plan (AC5), Design decisions Q18, and + `context/cli/mutation-trace-ref-reconciliation.md` describe the T04 regression + as exactly what it proves — the generic `WorktreeLock` happens-before ordering + that keeps reconciliation from inventorying/deleting while another owner holds + the lock and makes a pinned tree durable — and do **not** state that the T04 + test runs the production coordinator CAS. A stronger deterministic regression + exists and passes that pauses the real `coordinate()` after `pin_tree` / + `load_worktree` and before the real `store.commit` CAS (deterministic + channels/barriers, no sleeps as the synchronization mechanism, real + coordinator CAS path, reconciliation blocking on the real `WorktreeLock` and + retaining X, X's ref verified present afterward, the DB proving X became + durable through the coordinator flow), reached through the smallest + `pub(super)` coordinator test seam with no production behavior change. + - Validate: `services::mutation_trace::runtime::tests::reconciliation_blocks_until_a_real_coordinate_cas_commits_the_pinned_tree` (or the close-equivalent name) passes; inspection of AC5 / Q18 / the context doc wording; `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` stays clean (the seam is `#[cfg(test)]`/`pub(super)` only). +- [x] AC17: Every filesystem fixture in + `cli/src/services/mutation_trace/runtime/tests.rs` is owned by a + `tempfile::TempDir` whose `Drop` removes it; no test reaches a manual + `cleanup()` call to avoid leaving a stale top-level temporary directory, and + the `NEXT_ID` / `unique_path` / `SystemTime` / `UNIX_EPOCH` / `AtomicU64` / + `Ordering` / `cleanup` scaffolding is gone unless a specific remaining test + still needs it (with that need stated). Linked-worktree fixtures still drop + cleanly — via an explicit `git worktree remove` before fixture drop if a + supported platform needs it, with the outer filesystem lifecycle still + RAII-owned. + - Validate: `grep -n "fn cleanup\|unique_path\|AtomicU64\|UNIX_EPOCH\|NEXT_ID" cli/src/services/mutation_trace/runtime/tests.rs` returns nothing unaccounted for; `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::tests::` +- [x] AC18: The T09 integration suite exercises the clarified contracts end to + end, with the behavior under test always driven through `reconcile_worktree` / + `coordinate` (lower-level `GitSnapshotService` / raw-SQL helpers used only to + construct deliberately degraded prerequisite states), against real Git and a + real repository DB: active-worktree orphan deletion; current cursor retention + without a historical event; historical retention driven through real + `coordinate()` `A→B→C→D` transitions; idempotence (N then 0, stable + retained/root counts); linked-worktree isolation with no pause in the other + worktree; cross-worktree degraded-state retention with `git cat-file -t` still + resolving the tree; missing-local-required-pin fail-closed even when another + worktree pins the tree; a malformed / symbolic namespace ref fail-closed + leaving all refs untouched; a normal pass leaving every runtime-observable + DB / protocol field and the external-taint-marker state unchanged (only Git + refs differ — the migration-file guarantee is AC11's inspection, not a runtime + assertion); `git cat-file -t ` still resolving immediately after a + stale-ref delete (no `git gc` / `git prune` / `git reflog expire`); and the + missing-checkout-identity path returning + `ReconciliationOutcome::SkippedNoCheckoutIdentity` rather than a zero-count + `Reconciled` report. + - Validate: `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::tests::` (the T09 test set); the whole-suite command under Full validation + +### Full validation + +Repository-wide checks `/validate` runs after the last task. + +- `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::` +- `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` +- `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` +- `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check` +- `nix flake check` (runs `checks.cli-tests`, `checks.cli-clippy`, + `checks.cli-fmt`, `checks.mutation-trace-quint-connect`) +- `nix run .#pkl-check-generated` +- Confirm the existing Quint / Quint Connect model-based-testing harness + (`checks.cli-tests` / `checks.mutation-trace-quint-connect`) stays green; + **no `spec/mutation_cursor.qnt` behavior change is expected or made.** + +### Context sync + +- `context/cli/mutation-trace-store.md` — document the two new bounded + read-only queries: `load_tree_roots` (cursor + historical event trees for + one worktree, deduplicated) and `load_all_tree_roots` (the same three + `TreeId` columns unioned across every worktree, deduplicated, for the + reconciler's repository-wide retention set); state that **each query is + backed by a single SQL statement (`UNION` of `cursor_tree` / `before_tree` / + `after_tree`), so its full root set is read from one coherent database + snapshot** — not multiple independent `SELECT`s unioned in Rust — which is + what keeps a concurrent atomic `cursor T → X` + `event T → X` commit from + exposing a torn root set; update "Non-goals" so it no + longer implies the store exposes no durable-tree read for reconciliation + while keeping "no row deletion". +- `context/cli/mutation-trace-snapshot-service.md` — `GitSnapshotService::list_pins` + (returns `Result, PinInventoryError>`) / `delete_pins` are + documented here (the snapshot-service doc split out of the runtime-coordinator + doc during T02 context sync), including: **mutation-cursor pins are direct + refs; a symbolic ref inside the SCE mutation-cursor namespace is malformed + and rejected, never followed**, and **`delete_pins` uses + `git update-ref --no-deref --stdin` plus a fail-closed re-inventory pre-check + so an inventory→delete direct-ref→symref race cannot mutate a ref outside the + inventoried namespace**. +- `context/cli/mutation-trace-runtime-coordinator.md` — document the new + `runtime::ref_reconciliation` module (`reconcile_worktree`, the `pub(super)` + `reconcile_worktree_inner` test seam, `ReconciliationReport`, + `ReconcileError`, `RECONCILIATION_LOCK_TIMEOUT`, the fail-closed rules, and + the two-invariant model — per-worktree local consistency check + (`load_tree_roots`) vs. repository-wide deletion retention set + (`load_all_tree_roots`): an A-owned ref is retained whenever any worktree + still durably needs its tree; note that each of these reads is one SQL + statement / one DB snapshot, which is what makes the repository-wide read + safe against a concurrent atomic `cursor T → X` + `event T → X` commit + without a repository-global lock); + correct + the "one ref per pinned tree, **create-only**" on-disk-layout note to + "create-only per invocation; orphan/unreferenced pins reclaimed by the + per-worktree reconciliation pass, every pin for a current or historical + durable mutation-cursor root retained"; record that the `WorktreeLock` now + also guards reconciliation; extend the testing boundary. +- `context/cli/mutation-trace-protocol.md` — in "Target end-state + architecture", note that ref reconciliation is imperative durability + maintenance outside the verified protocol: it never advances the cursor, + chooses attribution, changes scope state, or creates a `MutationEvent`. +- `context/context-map.md` — refresh the `mutation-trace-runtime-coordinator.md` + and `mutation-trace-store.md` line annotations. +- `context/overview.md` — extend the `mutation_trace/runtime/` sentence to + mention the per-worktree ref-reconciliation maintenance pass. +- `spec/mutation_cursor.md` — under "Failure and durability boundary" / + "Implementation refinement", record that SCE-owned snapshot refs are + reclaimed by an imperative per-worktree maintenance pass (never modeled), + that Git performs object GC itself on its normal schedule, and that **SCE + deletes only its own refs, never Git objects directly**. +- `context/cli/mutation-trace-ref-reconciliation.md` (added during T03 context + sync) — (T05, broadened post-T05/T06 review) add the **unowned + checkout-identity limitation**: per-worktree reconciliation cleans only the + namespace of a checkout id a current worktree still derives; a namespace + becomes unreachable whenever no current worktree owns its checkout id — a + deleted linked worktree (Case A) or checkout-id metadata loss + recreation on + a still-present worktree (Case B) — so repository-scoped unowned-namespace + cleanup is future work, identity-ownership based, still governed by + `delete /T only if T ∉ durable_roots(repository)`. Preserve the + already-correct "reconciliation ≠ historical retention policy" statement. + (T06) replace "`read_checkout_id → Ok(None)` … is a clean no-op + `ReconciliationReport { 0, 0, 0 }`" with the + `ReconciliationOutcome::SkippedNoCheckoutIdentity` contract and its precise + meaning (no owned namespace inventoried; no DB comparison; no ref deleted; it + does **not** assert the repository holds no SCE refs). (T07) describe the T04 + regression as the `WorktreeLock` happens-before proof and record the exact + pin→CAS test's disposition. +- `context/cli/mutation-trace-runtime-coordinator.md` — (T06) update the + `reconcile_worktree` return-type mention to `ReconciliationOutcome` + (`Reconciled(ReconciliationReport)` | `SkippedNoCheckoutIdentity`); (T05, + broadened post-T05/T06 review) carry the same owned-checkout-identity-only / + unowned-namespace-future-work clarification; (T07) note the `pub(super)` + `after_load` coordinator test seam + exposed for the exact pin→CAS regression (test-only, no production behavior + change to `coordinate()`). +- Verify-only pass (expected: no edit): `context/architecture.md`, + `context/glossary.md`, `context/patterns.md`, + `context/cli/mutation-trace-external-taint.md` (reconciliation must not + touch the marker). + +## Task context synchronization lifecycle + +Persist this field in every plan; this is durable plan state, not chat state: + +- **Task context synchronization:** every task carries `pending | synced | blocked`. + A completed task must be `synced` before another task can start or the plan can + finish. +- For `blocked`, record **Blocker**, **Required action**, and **Retry condition** + beside the status. Never infer `synced` from conversation history; write every + lifecycle transition to the plan file. + +## Constraints and non-goals + +- **In scope:** + - `cli/src/services/mutation_trace/store.rs` — two new bounded, read-only + queries (`load_tree_roots` for one worktree, `load_all_tree_roots` for the + whole repository), each backed by **one SQL statement** (a single + `SELECT`/`UNION` constant covering `cursor_tree`, `before_tree`, and + `after_tree`) and **one** `query_map` call, plus its `TreeId` row mapper, + and inline tests. No write path, no schema, no migration change. + - `cli/src/services/mutation_trace/runtime/git_snapshot.rs` — new + worktree-scoped pin inventory (`list_pins`, returning `Result, + PinInventoryError>`) and conditional atomic deletion (`delete_pins`) + primitives on `GitSnapshotService`, plus a small `PinnedRef` value type + and the `PinInventoryError` enum, and inline tests. `capture_tree` / + `pin_tree` / `diff_trees` are unchanged. + - `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` (new; T06+) + — the per-worktree reconciliation algorithm, `ReconciliationReport`, + `ReconcileError`, the module-owned `RECONCILIATION_LOCK_TIMEOUT` constant, + the `pub fn reconcile_worktree` entrypoint (module-private to `runtime`, + like `coordinate`) and the `pub(super) fn reconcile_worktree_inner(.., + on_lock_contention)` test seam (visible only within `runtime`), and + inline tests. **T06** adds `pub enum ReconciliationOutcome { Reconciled(ReconciliationReport), SkippedNoCheckoutIdentity }` + and changes both function return types to `Result`. + - `cli/src/services/mutation_trace/runtime/mod.rs` — `mod ref_reconciliation;` + (private, matching `mod coordinator;`). + - `cli/src/services/mutation_trace/runtime/coordinator.rs` — **T07:** expose + the smallest `pub(super)` test seam that threads a real `after_load`-style + closure from `coordinate_inner` through `coordinate_protected` into the + existing `coordinate_boundary_inner` `after_load` parameter, so the T07 + regression can pause the real `coordinate()` after `pin_tree` / + `load_worktree` and before the real `store.commit` CAS. Test-only, no + production behavior change (production `coordinate()` keeps passing no-op + closures); no protocol / CAS / signature change to `coordinate()`. + - `cli/src/services/mutation_trace/runtime/tests.rs` — **T08** migrates the + existing manual-`cleanup()` / `unique_path` filesystem fixtures to a + RAII-owned `tempfile::TempDir` fixture; **T09** adds the cross-module + integration tests through `reconcile_worktree` / `coordinate` against real + Git repositories and a real repository-scoped Agent Trace DB. + - The durable context/spec docs named in "Context sync". +- **Out of scope:** harness adapters and any hook/command/`diff_traces` + wiring; a `pub(crate)` re-export of `reconcile_worktree`; deciding *when* + reconciliation runs (harness-wiring PR — see Design decisions Q15); + **implementing the repository-scoped unowned checkout-identity namespace + reconciliation operation** (recorded as future work only, Q16 — no + repository-global namespace scan or repository-global deletion is added by + this plan); + changing mutation-cursor protocol semantics, the Quint model, attribution, + the scope state machine, `MutationEvent` semantics, CAS semantics, + external-taint behavior, the DB schema, migrations, the durable-root + definition, the Git ref namespace format, `delete_pins` atomic-transaction + semantics, Git GC behavior, or automatic reconciliation scheduling; + auto-sync; control-plane changes; a daemon or background process; a new + persistent reconciliation bookkeeping table or cursor; repairing / recreating + a missing required ref (detect-and-fail-closed only, see Q7); a + repository-wide `refs/sce/**` scan or general-purpose ref cleanup; + cross-machine locking; a repository-global lock; `git gc` / `git prune` / + `git reflog expire` invocation; a generic Git abstraction redesign + (`capabilities::GitOps`); changing `pin_tree`'s create-only-per-invocation + behavior; changes to `protocol.rs`, `types.rs`, `store.rs`'s write path, + `spec/mutation_cursor.qnt`, or the Quint refinement matrix; new mutation + attribution rules; new `FailureKind`; new DB migration; the still-separate + `mutation-cursor-external-taint` concerns (that plan has landed). +- **Constraints:** + - No new **production** Cargo dependencies. Reconciliation reuses + `WorktreeLock` (`std::fs::File` advisory locks) via + `worktree_lock::acquire_inner` (already `pub(super)`), + `checkout::{resolve_git_dir, read_checkout_id}`, `GitSnapshotService`, + `MutationTraceStore`, and the `git` plumbing subprocess pattern already in + `git_snapshot.rs`. + `tempfile` is a dev-dependency used only for RAII-owned isolated test + directories (added by the mutation-trace store-test isolation fix and + reused by the `git_snapshot.rs` test-isolation fix). It is not linked into + production behavior and introduces no runtime dependency for + reconciliation. + - Reconciliation owns its own bounded lock timeout, + `const RECONCILIATION_LOCK_TIMEOUT: Duration = Duration::from_secs(10)` + in `ref_reconciliation.rs`. Its value intentionally matches the + coordinator's private `WORKTREE_LOCK_TIMEOUT` but is **not** a shared + abstraction — the coordinator constant stays private to `coordinator.rs`, + and neither is moved into `worktree_lock.rs` (there is no semantic reason + the two timeouts must always stay identical). + - New Git plumbing is limited to `git for-each-ref` (already in the + coordinator plan's validated command set, extended with a + `%(symref)` field so a symbolic ref in the namespace is detected and + rejected) and `git update-ref --no-deref --stdin` (T02 validates its + transaction and no-dereference semantics experimentally against this + repository's Git — `--no-deref` so a `delete` can never follow a symbolic + ref out of the inventoried namespace). No `git gc` / `git prune` / + `git reflog`. + - `git_snapshot.rs`'s ref namespace constant `REF_NAMESPACE` + (`refs/sce/mutation-cursor`) and `pin_ref_prefix` layout are the single + source of truth for the pin path; `pin_tree` / `list_pins` / `delete_pins` + derive their ref names from the same helper. Mutation-cursor pins are + **direct** refs only; a symbolic ref anywhere in the namespace is malformed + and rejected, never followed. + - The DB is supplied to `reconcile_worktree` by a caller-provided + `open_db: impl FnOnce() -> anyhow::Result` + provider, exactly like `coordinate()` — `reconcile_worktree` never + resolves repository identity or opens the DB itself. + - `reconcile_worktree` never accepts a caller-supplied `WorktreeId`, + `TreeId`, or ref name: worktree identity is derived from `repository_root` + → `resolve_git_dir` → `read_checkout_id`, exactly as `coordinate()` + derives it. + - Same-worktree pin inventory, root read, and deletion happen only while + that worktree's `WorktreeLock` is held. + - `cargo clippy` runs with `pedantic`/`warnings` denied workspace-wide. +- **Non-goal:** treating a reconciliation failure as mutation-cursor external + taint or any protocol failure. Reconciliation is maintenance: a failure to + delete an obsolete ref means storage cleanup did not complete, **not** that + mutation evidence is untrustworthy. It never arms `ExternalTaintMarker`, + calls `protocol::database_failure` / `protocol::taint`, mutates + mutation-cursor protocol state, produces a new `FailureKind`, changes + attribution, or triggers recovery. + +## Assumptions + +The user's change request states the example names/signatures are not +mandatory ("Do not prescribe this exact name or signature if the current +repository conventions suggest something better"). The following are recorded +local choices consistent with the existing `runtime/` conventions, not new +requirements. + +- The store queries are + `MutationTraceStore::load_tree_roots(&self, worktree: &WorktreeId) -> + Result>` and + `MutationTraceStore::load_all_tree_roots(&self) -> + Result>`, cold-path reads siblings to + `load_mutation_event` (which is likewise never called from `load_worktree` + or any hook-boundary path). Each method issues **exactly one SQL statement** + and **one** `query_map` call: a `UNION` of the three `TreeId` columns + (`mutation_trace_worktrees.cursor_tree`, `mutation_trace_events.before_tree`, + `mutation_trace_events.after_tree`) — `load_tree_roots` with each arm scoped + `WHERE worktree_id = ?1`, `load_all_tree_roots` with no `WHERE worktree_id` + clause — mapping the single `tree` column into `TreeId` and collecting + directly into `BTreeSet`. No other table contributes. The + constituent tables are **never** read with separate `query_map` calls and + unioned in Rust: the whole root set must come from one statement / one + database snapshot (see AC1 and "The challenge interleavings"). `UNION` is + chosen over `UNION ALL` because the API returns a set and duplicate + `TreeId`s are irrelevant; `UNION ALL` plus Rust `BTreeSet` dedup would be + acceptable only if the whole operation stays one SQL statement. A single + scope-parameterized query in place of the two methods would also be + acceptable if it fits repository conventions better — the two-method, + one-statement-each shape is the recorded local choice. The recorded SQL + constants are `SELECT_TREE_ROOTS_BY_WORKTREE_SQL` and + `SELECT_ALL_TREE_ROOTS_SQL` (not per-table constants such as + `SELECT_WORKTREE_CURSOR_ROOTS` / `SELECT_EVENT_ROOTS`). +- The Git pin-inventory primitive is + `GitSnapshotService::list_pins(&self, worktree_id: &WorktreeId) -> + Result, PinInventoryError>`, where + `PinnedRef { ref_name: String, tree: TreeId }` and + + ```rust + pub enum PinInventoryError { + /// `git for-each-ref` itself failed to execute or exited non-zero. + Git(anyhow::Error), + /// A ref under the SCE namespace is not shaped like a `pin_tree` output: + /// a symbolic ref, a non-tree target, a name/target SHA mismatch, an + /// unparseable `for-each-ref` line, or an unexpected extra path segment. + /// `reason` carries the specific discriminant for tests and `Display`. + MalformedRef { ref_name: String, reason: String }, + } + ``` + + This makes malformed SCE-namespace state separately matchable from a generic + `git for-each-ref` execution failure. `list_pins` uses the NUL-separated + `--format=%(refname)%00%(objectname)%00%(objecttype)%00%(symref)` and rejects + any ref with a non-empty `%(symref)` — mutation-cursor pins are direct refs. + The conditional-deletion primitive is + `GitSnapshotService::delete_pins(&self, pins: &[PinnedRef]) -> + anyhow::Result<()>` — it re-inventories the exact supplied ref names and + fails closed unless each is still a direct ref to a tree at the recorded SHA, + then runs one `git update-ref --no-deref --stdin` transaction. A pre-check + failure or a transaction failure (including a failed old-value check) is a + plain `Err` the reconciler maps to `ReconcileError::DeleteTransaction`. + `--no-deref` guarantees a `delete` can never follow a symbolic ref, so a + direct-ref→symref race between inventory and deletion cannot mutate a ref + owned by another worktree. +- The entrypoint is + `reconcile_worktree(repository_root: &Path, open_db: impl FnOnce() -> + anyhow::Result) -> Result` (T06 target; T03 shipped + `-> Result`), a `pub fn` in the private + `mod ref_reconciliation` (module-private to `runtime`, exactly like + `coordinate` in the private `mod coordinator` — never re-exported outside + mutation-trace `runtime`). It is a one-line delegation to + `pub(super) fn reconcile_worktree_inner(.., on_lock_contention: impl + FnOnce())` (mirroring `coordinate` / `coordinate_inner`, and matching + `worktree_lock::acquire_inner`'s existing `pub(super)`), with production + passing a no-op contention closure. `pub(super)` keeps the seam visible to + `runtime` and `runtime::tests` (where T04 lives) but invisible outside + `runtime` even if `ref_reconciliation` later becomes `pub`. +- `ReconciliationReport { local_required: usize, retained: usize, deleted: + usize }` (see Design decisions — Report shape). `local_required` is + `load_tree_roots(W).len()`; `retained` is `actual_W.len() − deleted`. + `retained == local_required` is **not** an invariant and the plan does not + claim it — a pin retained only because another worktree durably needs its + tree counts toward `retained` but not `local_required`. The only report + invariant, and it holds for a `Reconciled(report)` outcome only, is + `report.local_required ≤ report.retained`. `SkippedNoCheckoutIdentity` + carries no report, so no report invariant applies to it. +- (T06) The success type is `ReconciliationOutcome::Reconciled(ReconciliationReport) + | ReconciliationOutcome::SkippedNoCheckoutIdentity` (Design decisions Q17 — + Option A, an outcome enum, over Option B, a `status` field on the report), + chosen because `runtime` already prefers a matchable enum for this kind of + branch (`CoordinateError`, `CasResult`, `RuntimeBoundary`) and a skip has no + counts to report. The change request pre-authorized this local choice. + Current code truth: T03 shipped `Result`; + T06 delivers the outcome enum. +- `ReconcileError` is a matchable enum with `Display` + `std::error::Error` + (mirroring `CoordinateError`), with a distinct variant for **every** fallible + step and no `Other` catch-all: + + ```rust + pub enum ReconcileError { + GitDir(anyhow::Error), // resolve_git_dir + Lock(WorktreeLockError), // WorktreeLock acquisition + CheckoutIdentity(anyhow::Error), // read_checkout_id returned Err (corrupt/unreadable) + AgentTraceDbUnavailable(anyhow::Error), // open_db() provider failed + SnapshotService(anyhow::Error), // GitSnapshotService::new + PinInventory(anyhow::Error), // PinInventoryError::Git + MalformedPin { ref_name: String, reason: String }, // PinInventoryError::MalformedRef + DurableRoots(anyhow::Error), // load_tree_roots / load_all_tree_roots + MissingRequiredPins { missing: Vec }, // fail-closed (local consistency), nothing deleted + DeleteTransaction(anyhow::Error), // delete_pins + } + ``` + + `read_checkout_id() == Ok(None)` is **not** an error — it means + reconciliation has no current checkout identity from which to derive a + `WorktreeId` and its owned ref prefix (not a claim that none ever existed). + The reconciler has + already acquired `WorktreeLock(W)` by that point, and it releases the lock + and returns without creating an identity (no DB or Git-ref work); only `Err` + maps to `CheckoutIdentity`. **T03 returned a zero-count + `Ok(ReconciliationReport { 0, 0, 0 })` here; T06 replaces that with + `Ok(ReconciliationOutcome::SkippedNoCheckoutIdentity)` (Q17) so the skip is + observably distinct from a real zero-work pass.** + `AgentTraceDbUnavailable` here is a reconciliation + maintenance error only: it never arms `ExternalTaintMarker` and never + becomes `CoordinateError::AgentTraceDbUnavailable`, because no mutation + boundary is being coordinated. +- Test module/function names follow the crate module paths named in the + acceptance criteria. +- Reconciliation uses its own module-owned `RECONCILIATION_LOCK_TIMEOUT` + (`Duration::from_secs(10)` in `ref_reconciliation.rs`); it does not + reference the coordinator's private `WORKTREE_LOCK_TIMEOUT`. The values match + by intent, not by a shared constant. + +## Task stack + +- [x] T01: `Add worktree-scoped and repository-wide durable TreeId root queries to MutationTraceStore` (status:done) + - Task ID: T01 + - Scope: In — `cli/src/services/mutation_trace/store.rs`: + `pub fn load_tree_roots(&self, worktree: &WorktreeId) -> + Result>` and + `pub fn load_all_tree_roots(&self) -> Result>`, both + cold-path reads. Each is backed by **exactly one SQL statement** and + driven through **one** `query_map` call — not one `query_map` per backing + table with the results unioned in Rust. Also in — + `cli/src/services/db/mod.rs`: a `#[cfg(test)]` `pub(crate)` + `count_read_statements` seam plus a per-thread read-statement counter the + `TursoDb` read methods bump (test-only; no production behavior change). The + two recorded constants: + + - `SELECT_TREE_ROOTS_BY_WORKTREE_SQL` — + ```sql + SELECT cursor_tree AS tree FROM mutation_trace_worktrees WHERE worktree_id = ?1 + UNION + SELECT before_tree AS tree FROM mutation_trace_events WHERE worktree_id = ?1 + UNION + SELECT after_tree AS tree FROM mutation_trace_events WHERE worktree_id = ?1 + ``` + - `SELECT_ALL_TREE_ROOTS_SQL` — the same statement with **no** + `WHERE worktree_id` clause on any arm: + ```sql + SELECT cursor_tree AS tree FROM mutation_trace_worktrees + UNION + SELECT before_tree AS tree FROM mutation_trace_events + UNION + SELECT after_tree AS tree FROM mutation_trace_events + ``` + + Each statement maps its single `tree` column into `TreeId` and collects + directly into `BTreeSet`. `UNION` (set semantics) is intentional + because duplicate `TreeId`s are irrelevant to a set-typed API; + `UNION ALL` + Rust `BTreeSet` dedup is acceptable **only** if the whole + operation stays one SQL statement. An empty set (not `Err`) when the + scoped worktree has no `mutation_trace_worktrees` row and when the + repository has no rows at all. Read-only, cold path, no write changes, no + schema changes, no migration. Inline `#[cfg(test)] mod tests` cases + seeded via raw SQL (matching the existing `insert_worktree` / + `insert_mutation_event` test helpers in this file), covering + worktree-scoped exactness, repository-wide exactness, deduplication of a + tree shared by multiple worktrees, an empty repository, multiple + worktrees, **and the single-statement snapshot regression** described + below. A small test-only seam is added to + `cli/src/services/db/mod.rs`: `count_read_statements(body) -> (T, usize)` + (`#[cfg(test)]`, `pub(crate)`), backed by a per-thread counter each + `TursoDb` read method (`query` / `query_values` / `query_map`) bumps once + in its synchronous prelude, before the retry wrapper. No production + behavior change — the increments are `#[cfg(test)]`-only. + Out — any write path, `commit` change, schema/migration change, the Git + primitives (T02), the reconciliation algorithm (T03), calling either + query from anywhere (T03). + - Concurrency regression (in this task): two deterministic tests, no sleeps, + no probabilistic race, no stress loop. (a) + `load_all_tree_roots_retains_previous_cursor_after_atomic_cursor_advance` — + a **state-transition** test only: pre-advance `T` is a root via + `cursor_tree`, post-advance `T` is a root via `before_tree`. It is + explicitly *not* proof of snapshot isolation (a torn multi-read + implementation would also pass it). (b) + `load_all_tree_roots_reads_every_durable_root_in_one_sql_statement` and + `load_tree_roots_reads_every_durable_root_in_one_sql_statement` — the + **enforcement** regression. Each models the transition + `B.cursor_tree = T` → (atomically, one DB transaction) `cursor_tree := X` + **and** `INSERT mutation_trace_events { before_tree = T, after_tree = X }`, + then: (1) constructs the torn set that a multi-read implementation would + produce — read the event trees (empty), commit the atomic advance, read + the cursor trees (`{X}`), union in Rust → `{X}`, missing `T`; (2) asserts + the production `load_*_tree_roots` call, wrapped in + `count_read_statements`, issues **exactly one** read statement and returns + a set containing `T`. Reimplementing either query as two independent + `SELECT`s makes `count_read_statements` observe `2` and fails the test — + verified in-session by temporarily splitting `load_all_tree_roots` into + two `query_map` calls (`left: 2, right: 1`). The property this locks in: + one `load_*_tree_roots` invocation = one SQL statement = one DB snapshot, + which is the concurrency boundary because a single statement reads from one + coherent MVCC snapshot while two statements are two snapshots a concurrent + commit can fall between. + - Dependencies: none + - Done when: `load_tree_roots(W)` returns exactly `{cursor_tree(W)} ∪ + {before_tree, after_tree : mutation_trace_events row for W}`, deduplicated, + and `Ok(empty set)` for a worktree with no durable row; + `load_all_tree_roots()` returns the union of those same three columns + across **every** worktree, deduplicated, and `Ok(empty set)` for an empty + repository; **each method executes exactly one SQL statement through one + `query_map` call** — one `UNION` constant, no Rust-side union of per-table + result vectors — enforced at runtime by the + `…reads_every_durable_root_in_one_sql_statement` regression via + `count_read_statements`, not merely by inspection; neither query ever + returns a tree sourced from + `mutation_trace_scopes` / `mutation_trace_processed_events` / + `mutation_trace_event_active_scopes`, and `load_tree_roots` never returns + a tree belonging to another worktree; neither method is reachable from + `load_worktree` or a hook-boundary path (both are sibling cold-path reads + like `load_mutation_event`); the single-statement snapshot regression + passes. + - Verify: `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::store::`; + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings`; + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check`. + - Completed: 2026-08-31 + - Files changed: `cli/src/services/mutation_trace/store.rs`, + `cli/src/services/db/mod.rs` + - Result: Added two SQL constants — `SELECT_TREE_ROOTS_BY_WORKTREE_SQL` + (`?1`-scoped `UNION` of `cursor_tree` / `before_tree` / `after_tree`) and + `SELECT_ALL_TREE_ROOTS_SQL` (the same `UNION` with no `WHERE` clause) — + beside the existing `SELECT_*` constants, plus a free + `tree_root_row_from_turso(&turso::Row) -> Result` row mapper next + to the other `*_row_from_turso` functions. Added + `pub fn load_tree_roots(&self, worktree: &WorktreeId) -> Result>` + and `pub fn load_all_tree_roots(&self) -> Result>` on + `MutationTraceStore`, immediately after `load_mutation_event` (both + cold-path siblings, never reached from `load_worktree` or a hook-boundary + path). Each method is one `self.db.query_map(, .., tree_root_row_from_turso)` + call collecting `.into_iter().collect()` into `BTreeSet` — no + Rust-side union of per-table result vectors. turso accepts the reused `?1` + placeholder across the three `UNION` arms (verified by the passing tests). + In `db/mod.rs`, added a `#[cfg(test)]` statement-count seam: + `pub(crate) fn count_read_statements(body) -> (T, usize)` backed by a + per-thread `READ_STATEMENTS_ISSUED` cell that `TursoDb::{query, + query_values, query_map}` each bump once in their synchronous prelude + (before `run_with_retry_sync`, so retries never inflate the count). + Production builds are unaffected (`#[cfg(test)]`). + Added inline `#[cfg(test)] mod tests` cases plus local helpers + (`insert_worktree_with_cursor`, `insert_event_trees`, `tree_set`, + `apply_atomic_cursor_advance`, `select_trees`): the 8 exactness/dedup/empty + cases, the renamed state-transition test + `load_all_tree_roots_retains_previous_cursor_after_atomic_cursor_advance` + (pre `T` via `cursor_tree`, post `T` via `before_tree` across an atomic + `execute_transactional_cas_batch` advance; docstring states it is *not* + snapshot-isolation proof), and the two deterministic enforcement + regressions + `load_all_tree_roots_reads_every_durable_root_in_one_sql_statement` / + `load_tree_roots_reads_every_durable_root_in_one_sql_statement`, which + build the torn two-read set explicitly and assert the production path + issues exactly one read statement via `count_read_statements`. + - Verify (actual): `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::store::` + — 83 passed, 0 failed (10 new `load_*_tree_roots*` tests among them). + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::db` + — 22 passed (statement-count seam adds no regression). + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` — full + suite 855 passed, 0 failed. + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` + — no warnings. + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check` + — clean. + Regression proven: temporarily reimplementing `load_all_tree_roots` as two + `query_map` calls made + `load_all_tree_roots_reads_every_durable_root_in_one_sql_statement` fail + with `left: 2, right: 1`, while the state-transition test still passed — + confirming the counter test is the one that catches the torn-read + regression. Reverted. + - Deviations: The single-statement snapshot regression was strengthened from + the plan's original pre/post-only design (which the plan text now records + as insufficient) to the `count_read_statements` enforcement test, and the + weak test was renamed + `load_all_tree_roots_retains_previous_cursor_after_atomic_cursor_advance` + with an honest docstring. This required a `#[cfg(test)]`-only + read-statement counter in `cli/src/services/db/mod.rs` (added to scope + above) — the smallest deterministic seam; no production behavior change, + no new API on `load_tree_roots` / `load_all_tree_roots`. Recorded + assumption names and the two-method / one-statement-each shape were used + verbatim. Test fixtures seed events with + `attribution_kind = 'ineligible_unscoped'` / `boundary_kind = 'flush'` (the + minimal CHECK-satisfying shape) since only the tree columns matter here. + - Context impact: Domain. Adds two new public read-only methods to + `MutationTraceStore`'s contract (`load_tree_roots`, `load_all_tree_roots`); + no schema, migration, write-path, architectural, or cross-domain change. + Durable context to refresh per the plan's Context sync section: + `context/cli/mutation-trace-store.md` (document the two bounded queries and + the single-statement snapshot property; update "Non-goals"), and the + line annotations in `context/context-map.md`. No call site exists yet + (T03 wires the reconciler). + - Context synchronization: synced + +- [x] T02: `Add worktree-scoped pin inventory and conditional atomic deletion to GitSnapshotService` (status:done) + - Task ID: T02 + - Scope: In — `cli/src/services/mutation_trace/runtime/git_snapshot.rs`: a + `PinnedRef { ref_name: String, tree: TreeId }` value type; the + `PinInventoryError` enum (`Git(anyhow::Error)` for a `git for-each-ref` + execution/exit failure; `MalformedRef { ref_name: String, reason: String }` + for a non-tree target, a name/target SHA mismatch, an unparseable line, or + an unexpected extra path segment) with `Display` + `std::error::Error`; + `list_pins(&self, worktree_id: &WorktreeId) -> Result, + PinInventoryError>` running `git for-each-ref + --format=` constrained to the single path + prefix `refs/sce/mutation-cursor//` (derived from + `REF_NAMESPACE` plus a trailing `/`), parsing each line, and returning + `Err(PinInventoryError::MalformedRef { .. })` for any ref whose + `objecttype` is not `tree` or whose final path component does not equal its + `objectname`; `delete_pins(&self, pins: &[PinnedRef]) -> anyhow::Result<()>` + feeding one `git update-ref --stdin` transaction of `delete SP + SP LF` lines (a no-op returning `Ok(())` for an empty + slice), so the whole batch aborts and deletes nothing if any ref no longer + matches its expected value; a private test-only `delete_pins_inner(pins, + after_preflight)` seam (production `delete_pins` calls it with a no-op + hook) that fires `after_preflight` **after** the fail-closed preflight and + **before** the `git update-ref` transaction is spawned; inline + `#[cfg(test)] mod tests` extending the existing real-`git init` test + pattern in this file, including one test that pins two valid trees, passes + both through preflight, then uses the `after_preflight` hook to mutate the + second ref so the Git transaction is actually issued and its + expected-old-value check aborts the batch with every ref intact (the + canonical AC10 atomicity proof), plus tests matching + `PinInventoryError::MalformedRef` separately from `PinInventoryError::Git`. + Out — the reconciliation algorithm and `WorktreeLock` acquisition (T03); + the store query (T01); `capture_tree` / `pin_tree` / `diff_trees` changes; + any repository-wide ref enumeration. + - Dependencies: none + - Done when: `list_pins(W)` returns one `PinnedRef` per ref under exactly + `refs/sce/mutation-cursor//` and never a ref under another worktree's + prefix or any unrelated namespace; a ref in that namespace that is a + **symbolic ref**, whose target is not a tree, or whose name suffix + disagrees with its target SHA, makes `list_pins` return + `Err(PinInventoryError::MalformedRef { .. })`, matchable separately from + `PinInventoryError::Git(..)` (a `git for-each-ref` execution failure) — + **mutation-cursor pins are direct refs; a symbolic ref inside the SCE + mutation-cursor namespace is malformed and rejected, never followed**; + `delete_pins` removes exactly the supplied refs when every expected value + still matches, removes nothing and returns `Err` when any expected value + has changed **or an inventoried direct ref became a symbolic ref**, and is + a successful no-op for an empty slice; destructive ref operations use + `git update-ref --no-deref --stdin` (no-dereference semantics) so an + inventory→delete direct-ref→symref race cannot mutate a ref reached + through a symbolic ref (e.g. one owned by another worktree); the + `git update-ref --no-deref --stdin` transaction semantics relied on are + demonstrated by tests against this repository's Git. + - Verify: `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::git_snapshot::`; + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings`; + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check`. + - Completed: 2026-08-31 + - Files changed: `cli/src/services/mutation_trace/runtime/git_snapshot.rs` + - Result: Added `PinnedRef { ref_name: String, tree: TreeId }` + (`#[derive(Clone, Debug, Eq, PartialEq)]`) and the `PinInventoryError` enum + (`Git(anyhow::Error)`, `MalformedRef { ref_name: String, reason: String }`) + with a manual `Display` + `std::error::Error` impl, matching the + `CoordinateError` convention in `coordinator.rs`. Added a private + `pin_ref_prefix(worktree_id) -> String` helper + (`refs/sce/mutation-cursor//`) and routed the existing + `pin_ref_name` through it so the prefix has one source of truth. Added a + free `parse_pin_line(line, prefix) -> Result` + that enforces exactly-four NUL-separated `for-each-ref` fields, an **empty + `%(symref)`** (direct ref), an `objecttype` of `tree`, no extra path + segment after the worktree prefix, and refname-suffix equal to the target + SHA — every failure is `PinInventoryError::MalformedRef` with a + discriminating `reason`. Added + `GitSnapshotService::list_pins(&self, worktree_id: &WorktreeId) -> + std::result::Result, PinInventoryError>` running + `git for-each-ref` with the shared + `--format=%(refname)%00%(objectname)%00%(objecttype)%00%(symref)` constant + constrained to the single prefix (a `git for-each-ref` execution/exit + failure maps to `PinInventoryError::Git`), and + `GitSnapshotService::delete_pins(&self, pins: &[PinnedRef]) -> + anyhow::Result<()>` which first re-inventories the exact supplied ref names + in one order-independent `git for-each-ref` and fails closed unless each is + still a direct ref to a tree at the recorded SHA, then feeds one + `git update-ref --no-deref --stdin` transaction of + `delete SP SP LF` lines (empty slice is an + `Ok(())` no-op), spawned with the same `current_dir` + `GIT_DIR` env as + `run_git`. Production `delete_pins` is a thin wrapper over a private + `delete_pins_inner(pins, after_preflight: impl FnOnce())` that fires the + hook **after** `assert_pins_are_unchanged_direct_refs` and **before** the + `git update-ref --no-deref --stdin` transaction is spawned; production + passes a no-op hook, and only the inline atomicity test passes a real one. + `capture_tree` / `pin_tree` / `diff_trees` are unchanged. Added + 10 inline tests plus helpers (`other_worktree_id`, `capture_with_file`, + `ref_target`, `ref_exists`): prefix-scoped isolation, empty inventory, + `list_pins_rejects_a_ref_whose_target_is_not_a_tree`, + `list_pins_rejects_a_ref_whose_name_disagrees_with_its_target`, + `list_pins_rejects_a_symbolic_ref_inside_the_mutation_cursor_namespace` + (real `B/T -> T` direct + `A/T --symref--> B/T`; `list_pins(A)` returns + `MalformedRef` and `B/T` is unchanged), extra-path-segment rejection, the + `Git` variant on a removed git-dir, `delete_pins` exact removal, + empty-slice no-op, + `delete_pins_atomically_aborts_when_a_ref_changes_after_preflight` + (two valid direct refs, both pass preflight; the explicit + `[valid_ref, mismatched_ref]` batch is passed to `delete_pins_inner` with + an `after_preflight` hook that retargets only the second ref `B -> C`, so + `git update-ref --no-deref --stdin` is genuinely issued as + `delete valid_ref A` / `delete mismatched_ref B`, the second + expected-old-value check fails, Git commits neither delete, and + `valid_ref` is asserted still present at `A` — a sequential conditional + `git update-ref -d` implementation in input order would have committed the + first delete and failed this), and + `delete_pins_refuses_to_act_when_an_inventoried_direct_ref_became_a_symbolic_ref` + (inventory `A/T -> T` direct, then `A/T --symref--> B/T`; preflight makes + `delete_pins` return `Err` before any transaction, and both `A/T` and + `B/T` are untouched). + - Verify (actual): + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::git_snapshot::` + — 24 passed, 0 failed (10 pin tests). Broader + `services::mutation_trace::runtime::` — 67 passed, 0 failed. Full CLI suite + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` — 866 + passed, 0 failed. + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` + — no warnings. + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check` + — clean. Re-verified after the PR #246 round-2 regression fix + (`delete_pins_inner` seam + rewritten atomicity test): git_snapshot 24 + passed, `services::mutation_trace::runtime::` 67 passed, full CLI suite + 866 passed, clippy `--all-targets -D warnings` clean, fmt clean. + Re-verified after the PR #246 round-3 test-isolation fix (see the T02 + test-infrastructure follow-up in Deviations): git_snapshot 24 passed and + stable across 20 back-to-back parallel runs, `services::mutation_trace::` + 238 passed, clippy `--all-targets -D warnings` clean, fmt clean. + - Deviations: Recorded assumption names and signatures (`PinnedRef`, + `PinInventoryError` with its two variants, `list_pins` / `delete_pins` + signatures) were used verbatim. `list_pins` also rejects a ref with an + extra path segment after the worktree prefix. **Follow-up hardening (same + task, in response to a safety review of PR #246):** (1) the `for-each-ref` + format gained `%(symref)` (and switched to NUL-separated fields for + unambiguous parsing) so a **symbolic ref anywhere in the namespace is + rejected as `MalformedRef`, never followed** — a symref under worktree + `A`'s prefix could otherwise resolve through worktree `B`'s ref and be + accepted as a normal pin; (2) `delete_pins` now runs + `git update-ref --no-deref --stdin` (verified experimentally: without + `--no-deref`, `delete A/T ` on a symref `A/T -> B/T` deletes **both** + A/T and B/T; with `--no-deref` it can only ever remove the exact named + ref), plus a fail-closed re-inventory pre-check so the common + inventory→delete race returns a clean `Err` rather than acting on a symref. + The `Git`-variant test forces a `git for-each-ref` execution failure by + removing the resolved git-dir. **Follow-up regression fix (same task, PR + #246 review round 2):** the original + `delete_pins_aborts_the_whole_transaction_when_one_ref_no_longer_matches_its_expected_value` + mutated the second ref *before* calling `delete_pins`, so the fail-closed + preflight rejected the batch and `git update-ref` never ran — it proved + preflight, not Git transaction atomicity, and would have passed a + non-atomic sequential-delete implementation. Introduced the private + `delete_pins_inner(pins, after_preflight)` seam and rewrote the test as + `delete_pins_atomically_aborts_when_a_ref_changes_after_preflight`: both + refs pass preflight, the `after_preflight` hook then retargets the second + ref, the transaction is genuinely issued, and Git's expected-old-value + check aborts it with the first ref intact. The two properties now have + clearly separate proofs — preflight revalidation vs. the expected-old-value + atomic Git transaction. Production `delete_pins` signature and behavior are + unchanged (no-op hook). + **Test-infrastructure follow-up (same task, PR #246 review round 3):** + `git_snapshot.rs`'s test module still used a hand-rolled + PID/counter-derived temp path (`NEXT_TEST_REPO_ID` / `unique_test_repo`) + plus manual `remove_test_repo` cleanup, the same lifecycle weakness the + store tests shed in commit `6518c607` — a panicking test could leave a + directory a later process reuses. Replaced with an RAII-owned + `TestRepo { _temp_dir: tempfile::TempDir, root: PathBuf }` fixture and a + `test_repo(label)` constructor (`tempfile::Builder::prefix(..).tempdir()`); + every test now binds `let repo = test_repo(..); let repo_root = + repo.root().to_path_buf();` and holds `repo` (and its `TempDir`) for the + whole test, with no explicit cleanup call. `NEXT_TEST_REPO_ID`, + `unique_test_repo`, `remove_test_repo`, and the `AtomicU64`/`Ordering` + imports are gone. Tests are not serialized, use no sleeps/retries/mutex, + and the atomicity-test semantics are unchanged; `tempfile` was already a + dev-dependency. Production `git_snapshot.rs` (`capture_tree` / `pin_tree` + / `list_pins` / `delete_pins`) is untouched. + - Context impact: Domain. Adds new public items to `GitSnapshotService`'s + runtime-internal surface (`PinnedRef`, `PinInventoryError`, `list_pins`, + `delete_pins`); no schema, migration, protocol, marker, or cross-domain + change, and `git_snapshot` stays private to `mutation_trace::runtime`. No + call site exists yet (T03 wires the reconciler). Durable context to refresh + per the plan's Context sync section: + `context/cli/mutation-trace-runtime-coordinator.md` (document `list_pins` + returning `Result, PinInventoryError>` and `delete_pins`, + and extend the testing boundary). + - Context synchronization: synced + +- [x] T03: `Implement per-worktree reconciliation under WorktreeLock` (status:done) + - Task ID: T03 + - Scope: In — new + `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` and + `mod ref_reconciliation;` in `runtime/mod.rs`. `ReconciliationReport { + local_required, retained, deleted }`; the module-owned + `const RECONCILIATION_LOCK_TIMEOUT: Duration = Duration::from_secs(10)`; + `ReconcileError` with `Display` + `Error` and one variant per fallible + step, no `Other` catch-all: `GitDir(anyhow::Error)`, + `Lock(WorktreeLockError)`, `CheckoutIdentity(anyhow::Error)`, + `AgentTraceDbUnavailable(anyhow::Error)`, `SnapshotService(anyhow::Error)`, + `PinInventory(anyhow::Error)`, `MalformedPin { ref_name: String, reason: + String }`, `DurableRoots(anyhow::Error)`, `MissingRequiredPins { missing: + Vec }`, `DeleteTransaction(anyhow::Error)`. `pub fn + reconcile_worktree(repository_root, open_db)` (module-private to `runtime`) + delegating to `pub(super) fn reconcile_worktree_inner(repository_root, + open_db, on_lock_contention)`. Algorithm, with each step's error mapping, + entirely under the lock: + `resolve_git_dir(repository_root)` (`Err ⇒ GitDir`) → + `worktree_lock::acquire_inner(&git_dir, RECONCILIATION_LOCK_TIMEOUT, + on_lock_contention)` (`Err ⇒ Lock`) → + `checkout::read_checkout_id(&git_dir)`: `Ok(None) ⇒ return` a clean no-op + (the lock is already held; reconciliation has no current checkout identity + from which to derive a `WorktreeId` and its owned + `refs/sce/mutation-cursor//` prefix, so it runs no DB/Git-ref + work and creates no identity — see Q3/Q5). **T03 returned + `ReconciliationReport { 0, 0, 0 }` here; T06 returns + `ReconciliationOutcome::SkippedNoCheckoutIdentity` — the final contract.** + `Err ⇒ CheckoutIdentity` (a corrupt/unreadable id is **not** an + absent id); `Ok(Some(id)) ⇒ WorktreeId(id)` → + `open_db()` (`Err ⇒ AgentTraceDbUnavailable`; maintenance error only — + never arms the taint marker, never `CoordinateError`) → + `GitSnapshotService::new(repository_root)` (`Err ⇒ SnapshotService`) → + `actual = list_pins(&W)` (inventory **first**; + `Err(PinInventoryError::Git) ⇒ PinInventory`, + `Err(PinInventoryError::MalformedRef { ref_name, reason }) ⇒ MalformedPin { + ref_name, reason }`) → + `store = MutationTraceStore::new(&db)` → + `required_local = store.load_tree_roots(&W)` (`Err ⇒ DurableRoots`) → + `missing_local = required_local − {p.tree for p in actual}`; if non-empty ⇒ + `Err(MissingRequiredPins { missing: missing_local })` deleting nothing + (the **local consistency invariant** — a strictly per-worktree check, not + repository-wide) → + `required_repository = store.load_all_tree_roots()` (`Err ⇒ DurableRoots`) + → + `stale = [p in actual : p.tree ∉ required_repository]` (the **deletion + safety invariant** — an A-owned ref is removed only when no worktree in + the repository durably needs its tree); if empty ⇒ report with + `deleted: 0`; else `delete_pins(&stale)` (`Err ⇒ DeleteTransaction`) then + report `{ local_required: required_local.len(), retained: actual.len() − + stale.len(), deleted: stale.len() }`. Inline `#[cfg(test)] mod tests` + against a real temp-file `RepositoryAgentTraceDb` and a real + `GitSnapshotService` over a temp `git init` repo: orphan pin (with and + without a worktree row) deleted; current-cursor pin retained with no + referencing event; historical event before/after pins retained after the + cursor advances; a pin whose tree is absent from the target worktree's + roots but present in another worktree's durable rows (seeded via raw SQL) + is **retained** — repository-wide retention; `MissingRequiredPins` + fail-closed deleting nothing even when another worktree's row would cover + the missing tree; malformed namespace ref fail-closed deleting nothing; + idempotence; refs-deleted-without-object-reclamation; report counts + including the `retained > local_required` case. Out — the deterministic + lock-race regression (T04); cross-module + integration and linked-worktree tests (T09); any harness/command wiring; a + `pub(crate)` re-export; deciding invocation timing. + - Dependencies: T01, T02 + - Done when (T03 acceptance as authored; the result-type parts are superseded + by T06 per the annotation above — `Ok(None)` → `SkippedNoCheckoutIdentity`, + every other `Ok` → `Reconciled(ReconciliationReport { .. })`): + `reconcile_worktree` acquires the worktree's `WorktreeLock` + (via `worktree_lock::acquire_inner` with `RECONCILIATION_LOCK_TIMEOUT`) + before any pin or DB read and holds it until return; derives `WorktreeId` + only from `repository_root` (never a caller argument); distinguishes + `read_checkout_id` → `Ok(None)` (clean `{ local_required: 0, retained: 0, + deleted: 0 }` no-op, lock already held, no identity created) from `Err` + (`ReconcileError::CheckoutIdentity`); inventories pins before reading + durable roots; reads the target worktree's own roots (`load_tree_roots`) + for the fail-closed check and the repository-wide roots + (`load_all_tree_roots`) for the deletion decision; every fallible step + maps to its dedicated `ReconcileError` variant with no `Other` fallback; + fails closed with a distinct error and deletes nothing when any of the + **target worktree's** durable roots lacks a pin or the namespace contains + a malformed ref; otherwise deletes exactly the pins whose tree is outside + the **repository-wide** root set via one atomic `delete_pins` call and + returns a `ReconciliationReport` whose counts match; a checkout id present + but no durable row for W yields `{ local_required: 0, retained: R, + deleted: N }` where the retained `R` pins are exactly those W-owned pins + another worktree still durably needs and the deleted `N` are the rest, + with no error, and `{ 0, 0, 0 }` when there are also no pins; `open_db()` + failure is `AgentTraceDbUnavailable` and + provably never touches `ExternalTaintMarker`; + `reconcile_worktree_inner` is `pub(super)`; every listed inline test passes. + - Verify: `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::ref_reconciliation::`; + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings`; + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check`. + - Completed: 2026-09-01 + - Files changed: `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` + (new), `cli/src/services/mutation_trace/runtime/mod.rs` + - Result: Added `mod ref_reconciliation;` (private, beside `mod coordinator;`) + to `runtime/mod.rs`. New `ref_reconciliation.rs` contains: + `ReconciliationReport { local_required, retained, deleted }` + (`#[derive(Debug, Clone, Copy, PartialEq, Eq)]`); the module-owned + `const RECONCILIATION_LOCK_TIMEOUT: Duration = Duration::from_secs(10)`; + `ReconcileError` (manual `Display` + `std::error::Error`, matching + `CoordinateError`'s style — same-shaped `anyhow::Error` arms folded with + `|`) with exactly the ten recorded variants and no `Other`; `pub fn + reconcile_worktree(repository_root, open_db)` delegating to `pub(super) fn + reconcile_worktree_inner(repository_root, open_db, on_lock_contention)` + (mirroring `coordinate` / `coordinate_inner`, production passing a no-op + contention closure). The algorithm runs entirely under the lock in the + recorded order (result-type parts superseded by T06 — see the annotation + below): `checkout::resolve_git_dir` (`Err ⇒ GitDir`) → + `worktree_lock::acquire_inner(&git_dir, RECONCILIATION_LOCK_TIMEOUT, + on_lock_contention)` (`Err ⇒ Lock`) → `checkout::read_checkout_id` + (`Ok(None) ⇒` early `Ok(ReconciliationReport { 0, 0, 0 })` — T06 → early + `Ok(SkippedNoCheckoutIdentity)` — with the lock still held and no identity + created; `Err ⇒ CheckoutIdentity`; + `Ok(Some(id)) ⇒ WorktreeId(id)`) → `open_db()` (`Err ⇒ + AgentTraceDbUnavailable`, never arms the taint marker) → + `GitSnapshotService::new` (`Err ⇒ SnapshotService`) → `list_pins(&W)` + (`PinInventoryError::Git ⇒ PinInventory`, `PinInventoryError::MalformedRef + ⇒ MalformedPin { ref_name, reason }`) → `MutationTraceStore::new(&db)` → + `load_tree_roots(&W)` (`Err ⇒ DurableRoots`); `missing_local = + required_local.difference(pinned_trees)` non-empty ⇒ `Err(MissingRequiredPins + { missing })` deleting nothing → `load_all_tree_roots()` (`Err ⇒ + DurableRoots`) → `stale = actual.filter(|p| !required_repository.contains(&p.tree))`; + non-empty ⇒ `delete_pins(&stale)` (`Err ⇒ DeleteTransaction`) → `Ok` + report `{ local_required: required_local.len(), retained: actual.len() − + stale.len(), deleted: stale.len() }`. Inline `#[cfg(test)] mod tests`: a + RAII `Fixture` (`tempfile::TempDir`, real `git init` repo, checkout id via + `get_or_create_checkout_id`, schema DB via `RepositoryAgentTraceDb::new_at` + at a path **beside** the worktree, reopened per call through + `open_for_hooks_without_migrations_at`), plus `seed_worktree_cursor` / + `seed_event` raw-SQL helpers matching the store test helpers' column shape + (`attribution_kind = 'ineligible_unscoped'`, `boundary_kind = 'flush'`). + Ten tests: `orphan_pin_with_a_worktree_row_is_deleted`, + `orphan_pin_with_no_worktree_row_is_deleted`, + `current_cursor_pin_is_retained_without_a_referencing_event`, + `historical_event_before_and_after_pins_are_retained_after_the_cursor_advances` + (`A→B→C→D`, all four pins retained, `deleted: 0`), + `a_pin_another_worktree_durably_requires_is_retained` (repository-wide + retention **and** the `retained > local_required` count case: + `{ 0, 1, 1 }`), `a_missing_required_pin_fails_closed_and_deletes_nothing` + (local roots `{A, B}`, pins `{A, X}` → `MissingRequiredPins { missing: + [B] }`, both refs intact), `a_malformed_namespace_ref_fails_closed_and_deletes_nothing` + (a `git symbolic-ref` inside the namespace → `MalformedPin`, nothing + deleted), `reconciliation_is_idempotent`, + `reconciliation_deletes_refs_without_reclaiming_objects` (`git cat-file -t` + still resolves the orphan tree immediately after its ref is deleted), and + `no_checkout_identity_is_a_clean_no_op` (checkout-id file removed → + `Ok({ 0, 0, 0 })`, the pin left untouched). + - **Superseded for the final runtime result contract by T06:** T03 shipped + `reconcile_worktree(..) -> Result` + and returned `Ok(ReconciliationReport { 0, 0, 0 })` on the + `read_checkout_id → Ok(None)` branch. T06 changes the return type to + `Result` and returns + `Ok(ReconciliationOutcome::SkippedNoCheckoutIdentity)` there (and + `Reconciled(..)` elsewhere), so the two states — "a real pass found no + work" and "no pass ran" — are observably distinct (Q17, AC14). The + `no_checkout_identity_is_a_clean_no_op` test above is replaced by + `no_checkout_identity_returns_a_distinct_skipped_outcome` in T06. This + annotation records the evolution; the T03 evidence above is not rewritten. + - Verify (actual): + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::ref_reconciliation::` + — 10 passed, 0 failed. + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::` + — 248 passed, 0 failed. + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` — full + suite 878 passed, 0 failed. + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` + — no warnings (`reconcile_worktree` / `reconcile_worktree_inner` are + consumed only by the inline tests for now, exactly as `coordinate` is; no + dead-code warning, matching that precedent). + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check` + — clean. + - Deviations: Recorded assumption names and signatures were used verbatim. + The `resolve_git_dir` used for the lock and checkout-id read is + `crate::services::checkout::resolve_git_dir` (relative `.git` resolved + against `repository_root`), exactly as `coordinate()` derives identity — + `GitSnapshotService::new` still resolves its own git-dir internally. Added + one test beyond the plan's enumerated list, + `no_checkout_identity_is_a_clean_no_op`, to cover the `read_checkout_id → + Ok(None)` branch named in "Done when" (clean `{ 0, 0, 0 }` no-op with the + lock already held and no identity created). No production behavior beyond + the reviewed task. + - Context impact: Domain. Adds a new `runtime::ref_reconciliation` module + (`reconcile_worktree`, `reconcile_worktree_inner`, `ReconciliationReport`, + `ReconcileError`, `RECONCILIATION_LOCK_TIMEOUT`) to the mutation-trace + runtime's internal surface; the module is private to + `mutation_trace::runtime` and not re-exported. No schema, migration, + protocol, marker, spec, Quint, or cross-domain change; no new production + dependency; no `mutation_trace_*` write. Durable context to refresh per the + plan's Context sync section: + `context/cli/mutation-trace-runtime-coordinator.md` (document the new + module, the two-invariant model, `RECONCILIATION_LOCK_TIMEOUT`, the + fail-closed rules, the `pub(super)` seam; correct the "create-only" on-disk + layout note; record that `WorktreeLock` now also guards reconciliation; + extend the testing boundary), `context/cli/mutation-trace-protocol.md` + ("Target end-state architecture" — reconciliation is imperative durability + maintenance outside the verified protocol), `context/context-map.md` and + `context/overview.md` (line annotations / the `mutation_trace/runtime/` + sentence), and `spec/mutation_cursor.md` ("Failure and durability + boundary" / "Implementation refinement"). No call site outside the module + yet (T04 / T07 / T09 add tests through the seam; harness wiring is a later + PR). + - Context synchronization: synced + +- [x] T04: `Add the deterministic pin-to-CAS synchronization regression` (status:done) + - Task ID: T04 + - Scope: In — `cli/src/services/mutation_trace/runtime/tests.rs` (a child of + `runtime`, so it can reach the `pub(super)` seam): one deterministic + concurrency test that (a) acquires a real `WorktreeLock` for worktree W on + the main thread via `worktree_lock::acquire_inner`; (b) spawns a worker + calling `ref_reconciliation::reconcile_worktree_inner` for W with an + `on_lock_contention` closure that signals a `std::sync::mpsc` channel; (c) + waits (bounded) on that channel for the contention signal and asserts the + worker has **not** completed while the lock is held; (d) still holding the + lock, pins a tree X via `GitSnapshotService` and makes X a durable root (a + committed baseline / `initialize_worktree` + cursor at X, or a real prior + `coordinate()` whose lock is then re-taken by the test); (e) drops the + lock; (f) joins the worker and asserts it returns `Ok` with `deleted == 0` + and X's ref still present. Reuse the + `two_threads_on_the_same_worktree_serialize` structure already in this + file. Out — any production-code change (the `pub(super)` seam already + exists from T03); the retained-root / linked-worktree / no-write scenarios + (T09); the exact real-coordinator pin→CAS regression (T07). + - Dependencies: T03 + - Done when: the test proves — via the `WorktreeLock` happens-before edge, + with no `sleep`-based timing — that a reconciliation pass blocks for the + entire interval another holder owns the lock, and that once it proceeds it + observes X among the durable roots and retains X rather than deleting it; + `services::mutation_trace::runtime::` passes. + - Verify: `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::tests::reconciliation_blocks_on_the_worktree_lock`; + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::`. + - Completed: 2026-09-01 + - Files changed: `cli/src/services/mutation_trace/runtime/tests.rs` + - Result: Added the deterministic AC5 regression + `reconciliation_blocks_on_the_worktree_lock_and_retains_a_pin_that_becomes_durable_under_it` + to `runtime/tests.rs`, plus a local `seed_event` raw-SQL helper mirroring + the `ref_reconciliation.rs` / store helpers (`attribution_kind = + 'ineligible_unscoped'`, `boundary_kind = 'flush'`). Four `use` items added: + `std::sync::mpsc`, `super::worktree_lock::acquire_inner`, + `super::ref_reconciliation::reconcile_worktree_inner`, and + `crate::services::mutation_trace::store::encode_revision`. The test runs a + real `coordinate()` Flush baseline (establishing checkout identity and a + pinned durable cursor row), then on the main thread takes the worktree's + `WorktreeLock` via `acquire_inner`; a worker thread calls + `reconcile_worktree_inner` with an `on_lock_contention` closure that signals + an `mpsc` channel. The test waits (bounded, `recv_timeout`) for the + contention signal, asserts via a 300 ms negative `recv_timeout` that the + pass has not completed while the lock is held, then — still holding the + lock — captures a fresh tree `X` (writes `under-lock.txt`), `pin_tree`s it, + and seeds a `mutation_trace_events` row (`before = baseline tree`, `after = + X`) making `X` a repository-wide durable root. It drops the lock, joins the + worker, and asserts `Ok` with `deleted == 0`, `local_required == 2`, and + `refs/sce/mutation-cursor//` still resolvable via `git show-ref + --verify`. No `sleep`; the proof is the `WorktreeLock` happens-before edge. + No production code changed. + - Verify (actual): + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::tests::reconciliation_blocks_on_the_worktree_lock` + — 1 passed, 0 failed. + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::` + — 78 passed, 0 failed. + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` + — no warnings. + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check` + — clean. + - Deviations: Per a user instruction during implementation, no explanatory + code comments were added (the test is comment-free). The + `two_threads_on_the_same_worktree_serialize` structure reused for the + channel/timeout pattern lives in `coordinator.rs`, not `tests.rs` as the + scope note says — same `runtime` module tree. Durable identity and the + pre-existing local durable root were established with a real `coordinate()` + baseline (one of the plan's offered options); `X` was made a durable root + by a raw-SQL `mutation_trace_events` insert, the sibling-test convention. + - Context impact: None. Test-only addition to + `cli/src/services/mutation_trace/runtime/tests.rs`; no production code, + signature, schema, migration, protocol, marker, spec, Quint, or + cross-domain change; no new dependency. The `runtime::ref_reconciliation` + surface documented under this plan's Context sync section is unchanged by + this task — T03 already recorded the module's durable-context needs, and + T09 completes the integration-test coverage. No durable context file needs + an edit for T04. + - Context synchronization: synced + +- [x] T05: `Record the retired-worktree namespace limitation and future repository-scoped reconciliation` (status:done) + - Task ID: T05 + - Scope: In — durable documentation only. + `context/cli/mutation-trace-ref-reconciliation.md` and + `context/cli/mutation-trace-runtime-coordinator.md`: state precisely that + per-worktree reconciliation reclaims orphan / unreferenced refs **only for + a still-identifiable current worktree namespace**; that a deleted / retired + linked worktree loses its `/sce/checkout-id` while its + `refs/sce/mutation-cursor//*` refs remain in the shared + repository; that no surviving worktree can derive that old namespace, so the + current per-worktree reconciler structurally cannot reach it; and that + repository-scoped retired-worktree cleanup is **future work**. Record the + future operation's shape (repository namespace enumeration → active vs. + retired checkout ids → per-retired-namespace + tree-vs-`durable_roots(repository)` comparison → delete only non-durable + trees; "no active worktree → delete whole namespace" forbidden; same + false-retention-over-false-deletion bias). Clarify the harness gate: + high-frequency harness traffic against active worktrees is handled by this + pass; ephemeral-worktree deletion is the separate future lifecycle. Correct + any wording in these docs (and cross-check `context/overview.md` / + `context/context-map.md` annotations) that implies the current pass prevents + **all** orphan-ref accumulation. Preserve the already-correct + "reconciliation ≠ historical retention policy" statement. Out — any code + change; implementing the future operation; the skipped-outcome wording + (T06); the T04 reframing (T07). + - Dependencies: none (plan already carries the "Active-worktree scope" + section and Design decisions Q16 authored with this revision) + - Done when: both named context docs describe the active-worktree scope and + the retired-worktree limitation in the terms above, name the future + repository-scoped operation, and contain no claim that the current pass + bounds all orphan-ref growth; `nix run .#pkl-check-generated` and + `nix flake check` stay green (doc-only change). + - Verify: `grep -n "retired\|still-identifiable\|repository-scoped" context/cli/mutation-trace-ref-reconciliation.md context/cli/mutation-trace-runtime-coordinator.md`; + `nix run .#pkl-check-generated`. + - Completed: 2026-09-01 + - Files changed: `context/cli/mutation-trace-ref-reconciliation.md`, + `context/cli/mutation-trace-runtime-coordinator.md`, `context/overview.md`, + `context/context-map.md` + - Result: Added a `## Scope: an active, still-identifiable worktree namespace + only` section to `mutation-trace-ref-reconciliation.md` (after "Entry point + and identity") covering: the owned prefix derived from the current + worktree's checkout id; `/sce/checkout-id` disappearing on + `git worktree remove` while `refs/sce/mutation-cursor//*` + survive in the shared namespace; no surviving worktree being able to derive + a retired id (a stranded-namespace text diagram); the harness gate + (persistent / current-worktree lifecycle is storage-cleanup complete, + ephemeral-linked-worktree lifecycle is not until the future op exists); and + a `### Future work: repository-scoped retired-worktree reconciliation` + subsection recording the operation shape (namespace enumeration → active vs. + retired ids → per-retired-namespace tree-vs-`durable_roots(repository)` + comparison → delete only non-durable) with inherited invariants (repo-wide + durability via `load_all_tree_roots()`, false-retention bias, "no active + worktree → delete whole namespace" forbidden). Tightened the doc's opening + sentence to "within one still-identifiable worktree's ref namespace" and + added a cross-reference; preserved the existing "not a bound on storage + growth" / historical-retention statement and tied the two scope limits + together explicitly. In `mutation-trace-runtime-coordinator.md`, extended + the `ref_reconciliation.rs` module bullet with the active-worktree-only + scope, the retired-worktree future-work note, and an explicit "does not + bound all orphan-ref growth" statement, and updated the on-disk-layout + annotation for `refs/sce/mutation-cursor//`. Added a + one-clause precision qualifier to the `ref_reconciliation` mentions in + `context/overview.md` and `context/context-map.md`. No code, spec, or Quint + change. + - Verify (actual): + `grep -n "retired\|still-identifiable\|repository-scoped" context/cli/mutation-trace-ref-reconciliation.md context/cli/mutation-trace-runtime-coordinator.md` + — matches in both files (scope section, future-work subsection, module + bullet, on-disk-layout note). + `nix run .#pkl-check-generated` — "Ephemeral Pkl generation passed: 141 + files", inventory sha256 unchanged. + `nix flake check` — "all checks passed!". + - Deviations: Also touched `context/overview.md` and `context/context-map.md` + (named in the task as "cross-check" targets, not in the two-doc "In" list) — + each got a single precision clause so their `ref_reconciliation` annotations + no longer read as covering every namespace. The stale "T04/T05 tests" + references inside `mutation-trace-ref-reconciliation.md` were left untouched: + reframing the T04 regression wording is explicitly T07's scope. + - Context impact: Documentation-only. This task *is* durable context work — it + directly satisfies the T05 items in this plan's "Context sync" section + (`context/cli/mutation-trace-ref-reconciliation.md` retired-worktree + limitation; `context/cli/mutation-trace-runtime-coordinator.md` + active-worktree-only clarification) plus the cross-checked + `overview.md` / `context-map.md` annotations. No production code, signature, + schema, migration, protocol, marker, spec, or Quint change; no new + dependency. Root-context files: `context/overview.md` and + `context/context-map.md` updated as above; `context/architecture.md`, + `context/glossary.md`, `context/patterns.md` need no change (no new module, + term, or pattern — scope-limitation wording only). + - Post-completion review fix (docs only, no production semantic change): the + limitation was broadened from "retired / deleted linked worktree namespace" + to **unowned checkout-identity namespace** — the fundamental unsupported + lifecycle is "an SCE ref namespace exists and no current worktree owns / + derives that checkout id", of which a deleted linked worktree is **Case A** + and checkout-id metadata loss followed by `get_or_create_checkout_id` + minting a fresh id (leaving the old namespace unowned while the physical + worktree still exists) is **Case B** — a metadata-loss / recreation + lifecycle, not normal operation. The future repository-scoped operation now + reasons about checkout-identity ownership rather than worktree-path deletion, + still `delete /T only if T ∉ durable_roots(repository)`, still + forbidding "unowned → delete the whole namespace". Harness wording: + "persistent worktree with a **stable** checkout identity" is covered by the + current pass; "any lifecycle that can retire, replace, lose, or recreate + checkout identities" needs the future op. Applied to this plan's "Scope" + section + Q15/Q16/Q17, the change-summary, AC13/AC15, the "Open questions" + summary, `context/cli/mutation-trace-ref-reconciliation.md`, + `context/cli/mutation-trace-runtime-coordinator.md`, and the `overview.md` / + `context-map.md` annotations. The stale "T04/T05 tests" wording deferred to + T07 was resolved separately during T07. + - Context synchronization: synced + +- [x] T06: `Make missing checkout identity an observable skipped outcome` (status:done) + - Task ID: T06 + - Scope: In — `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs`: + add `pub enum ReconciliationOutcome { Reconciled(ReconciliationReport), + SkippedNoCheckoutIdentity }` (deriving the same `Debug, Clone, Copy, + PartialEq, Eq` as `ReconciliationReport`); change `reconcile_worktree` and + `reconcile_worktree_inner` return types to + `Result`; on the `read_checkout_id → + Ok(None)` branch return `Ok(ReconciliationOutcome::SkippedNoCheckoutIdentity)` + (still with `WorktreeLock` released on return and no identity created) and + on the normal path wrap the report in `Reconciled(..)`. Update every inline + `#[cfg(test)] mod tests` assertion in this file (the existing tests assert + on a bare `ReconciliationReport`; adapt via a small + `outcome.expect_reconciled()`-style test helper or explicit `match`). + Replace the current `no_checkout_identity_is_a_clean_no_op` test with + `no_checkout_identity_returns_a_distinct_skipped_outcome` (asserts + `SkippedNoCheckoutIdentity`, **not** a zero report) and add + `a_missing_checkout_identity_skip_touches_no_db_and_no_ref` (an `open_db` + provider that panics if invoked; a pre-seeded ref still byte-identical + after the skip; keep the assertion structural — no invasive production + seam). Also in — `cli/src/services/mutation_trace/runtime/tests.rs`: update + the T04 test (`reconciliation_blocks_on_the_worktree_lock_and_retains_a_pin_that_becomes_durable_under_it`), + which reads `report.deleted` / `report.local_required`, to unwrap + `Reconciled(..)` first. Out — the RAII fixture migration (T08); new + integration scenarios (T09); the retired-worktree docs (T05); the T04 + reframing (T07); any change to `ReconciliationReport`'s fields, the two + invariants, `delete_pins`, or the store queries. + - Dependencies: T05 + - Done when: `reconcile_worktree(..)` returns + `Result`; a real zero-work pass + returns `Reconciled(ReconciliationReport { .., deleted: 0 })` and a + no-checkout-identity pass returns `SkippedNoCheckoutIdentity` (an `Ok`, + never an `Err`); the skip invokes no `open_db`, inventories no pins, and + leaves every SCE ref untouched; `services::mutation_trace::runtime::` + passes; clippy `-D warnings` clean; `fmt --check` clean. + - Verify: `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::`; + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings`; + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check`. + - Completed: 2026-09-01 + - Files changed: `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs`, + `cli/src/services/mutation_trace/runtime/tests.rs` + - Result: Added `pub enum ReconciliationOutcome { Reconciled(ReconciliationReport), + SkippedNoCheckoutIdentity }` (same `Debug, Clone, Copy, PartialEq, Eq` + derives as `ReconciliationReport`) to `ref_reconciliation.rs`. Changed + `reconcile_worktree` and `reconcile_worktree_inner` return types to + `Result`. The `read_checkout_id → + Ok(None)` branch now returns `Ok(ReconciliationOutcome::SkippedNoCheckoutIdentity)` + (lock still released on return, no identity created, no `open_db` call, no + inventory) instead of constructing a zeroed `ReconciliationReport`; the + normal path wraps its report in `Reconciled(..)`. Inline tests adapt via a + new private `expect_reconciled(outcome) -> ReconciliationReport` helper. + Replaced `no_checkout_identity_is_a_clean_no_op` with + `no_checkout_identity_returns_a_distinct_skipped_outcome` (asserts + `SkippedNoCheckoutIdentity`, not a zero report) and added + `a_missing_checkout_identity_skip_touches_no_db_and_no_ref` (an `open_db` + provider that panics if invoked; the pre-seeded ref's `git rev-parse` output + byte-identical before and after the skip). In `runtime/tests.rs`, imported + `ReconciliationOutcome` and unwrapped `Reconciled(..)` in the T04 test + (`reconciliation_blocks_on_the_worktree_lock_and_retains_a_pin_that_becomes_durable_under_it`) + before reading `report.deleted` / `report.local_required`. No change to + `ReconciliationReport`'s fields, the two invariants, `delete_pins`, or the + store queries. + - Verify (actual): + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::` + (via `nix develop -c`) — `ok. 79 passed; 0 failed`, including the two new + tests and the reframed skip test. + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` + — `Finished` with no warnings. + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check` + — clean (no diff, exit 0). + - Deviations: None. The user asked mid-task to keep comment churn minimal, so + the reverted `// Clean no-op; no identity is created.` inline comment on the + `None` branch is left verbatim from before this task and only a one-line doc + comment was added to the new public `ReconciliationOutcome` enum (consistent + with `ReconciliationReport` / `ReconcileError`). + - Context impact: New public type `ReconciliationOutcome` and a changed return + type on `runtime::ref_reconciliation::{reconcile_worktree, + reconcile_worktree_inner}` — both module-private to `runtime`, no + re-export, no caller outside `runtime`. Durable-context follow-ups already + enumerated in this plan's "Context sync" section (T06 items): + `context/cli/mutation-trace-ref-reconciliation.md` (replace the + `ReconciliationReport { 0, 0, 0 }` no-op wording with the + `SkippedNoCheckoutIdentity` contract) and + `context/cli/mutation-trace-runtime-coordinator.md` (update the + `reconcile_worktree` return-type mention to `ReconciliationOutcome`). No + schema, migration, protocol, marker, spec, or Quint change; no new + dependency. Root-context files: `context/context-map.md` line annotation + for `mutation-trace-ref-reconciliation.md` refreshed to name the + `ReconciliationOutcome` return type; `context/overview.md` describes the + pass behaviorally and needs no change; `context/architecture.md`, + `context/glossary.md`, `context/patterns.md` unaffected. + - Post-completion review fix (test + doc hardening, no production semantic + change; `reconcile_worktree` control flow, skip ordering, and all field + shapes unchanged): + (1) `ref_reconciliation.rs` — the `ReconciliationReport` doc comment's stale + "the only relation that holds on the `Ok` path is `local_required <= + retained`" (inaccurate once `Ok(SkippedNoCheckoutIdentity)` carries no + report) is now per-variant: `report.local_required <= report.retained` for + `Reconciled(report)`, and "`SkippedNoCheckoutIdentity` carries no report, so + no report invariant applies". The same stale sentence in + `context/cli/mutation-trace-ref-reconciliation.md` was fixed to match. + (2) `a_missing_checkout_identity_skip_touches_no_db_and_no_ref` — the + `git rev-parse ` before/after comparison (which only pins the resolved + SHA) is replaced with a test-local `git for-each-ref + --format=%(refname)%00%(objectname)%00%(objecttype)%00%(symref)` capture, so + the skip is now proven to leave the ref's **name, target SHA, object type, + and direct/symbolic shape** all unchanged — a direct ref silently turning + symbolic while still resolving to the same SHA now fails. The + panic-on-`open_db` provider assertion is preserved unchanged. AC15 updated + to describe the structural check. + - Context synchronization: synced + +- [x] T07: `Add the exact real-coordinator pin→CAS reconciliation regression` (status:done) + - Task ID: T07 + - Scope: In — + (1) `cli/src/services/mutation_trace/runtime/coordinator.rs`: expose the + **smallest** `pub(super)` test seam that threads a real `after_load`-style + closure from `coordinate_inner` (currently a private `fn`) through + `coordinate_protected` (currently hardcodes `|_attempt| {}` for `after_load`) + into the existing `coordinate_boundary_inner` `after_load` parameter. + Production `coordinate()` keeps passing no-op closures — no production + behavior change, no new Git abstraction / runtime capability layer / + callback framework, no protocol or CAS change. + (2) `cli/src/services/mutation_trace/runtime/tests.rs`: add + `reconciliation_blocks_until_a_real_coordinate_cas_commits_the_pinned_tree` + (or a very close equivalent): a `coordinate()` baseline initializes the + worktree; a worker thread runs a real `coordinate(Flush)` whose `after_load` + hook signals a channel and blocks on a barrier (pin X already done, real + `store.commit` CAS not yet); a reconciliation worker then runs through + `reconcile_worktree` / `reconcile_worktree_inner` and must block on the real + `WorktreeLock`; releasing the barrier lets the coordinator run the real CAS + that commits X and drop the lock; reconciliation then acquires the lock, + reads `load_all_tree_roots`, and retains X. Assert: reconciliation observed + contention; X's `refs/sce/mutation-cursor//` ref survives; the DB shows + X durable via the real coordinator flow (`mutation_trace_events` / + `cursor_tree`). Deterministic channels/barriers only — no sleeps as the + synchronization mechanism. + (3) plan Design decisions Q18 + `context/cli/mutation-trace-ref-reconciliation.md`: + describe T04 as the generic `WorktreeLock` happens-before proof and T07 as + the same property across the real `capture → pin → load → prepare → store + CAS` path; note the `pub(super)` seam (test-only, no production behavior + change). T04 is **kept**, not removed. + Out — a large abstraction or `capabilities::GitOps`-style redesign; any + production coordinator behavior change; the outcome enum (T06); the RAII + migration (T08); the T09 scenarios. + - Dependencies: T06 + - Done when: `reconciliation_blocks_until_a_real_coordinate_cas_commits_the_pinned_tree` + passes and proves — with no sleep-based synchronization — that a + reconciliation pass blocks for the whole interval the real `coordinate()` + holds the `WorktreeLock` across `pin X → real CAS`, then retains X; the + coordinator seam is `pub(super)` / `#[cfg(test)]`-reachable only and + `coordinate()`'s production signature and behavior are unchanged; AC5 / Q18 / + the context doc state precisely what T04 vs T07 prove; + `services::mutation_trace::runtime::` passes; clippy `-D warnings` and + `fmt --check` clean. + - Verify: `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::`; + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings`; + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check`. + - Completed: 2026-09-01 + - Files changed: `cli/src/services/mutation_trace/runtime/coordinator.rs`, + `cli/src/services/mutation_trace/runtime/tests.rs`, + `context/cli/mutation-trace-ref-reconciliation.md` + - Result: + (1) `coordinator.rs` — added an `after_load: L` (`L: FnMut(u32)`) parameter to + `coordinate_inner` and `coordinate_protected`, threading it into the existing + `coordinate_boundary_inner` `after_load` slot in place of the hardcoded + `|_attempt| {}`. Made `coordinate_inner` `pub(super)` (reachable from + `runtime` / `runtime::tests`, invisible outside `runtime`, mirroring + `reconcile_worktree_inner`). + Production `coordinate()` now passes `|_attempt| {}` for `after_load` — no + behavior change, no protocol / CAS / signature change. Updated the three + existing in-file `coordinate_inner` test call sites with the no-op + `after_load`. + (2) `runtime/tests.rs` — added + `reconciliation_blocks_until_a_real_coordinate_cas_commits_the_pinned_tree` + (`#[allow(clippy::too_many_lines)]`, matching the three existing such tests in + the file). A baseline `coordinate(Flush)` materializes the worktree; a real + edit is written; a worker runs `coordinate_inner(Flush, .., after_load, + ..)` whose `after_load` hook (a `move` `FnMut` guarded by a `paused` flag) + signals a channel then blocks on a second channel — pin X done, real + `store.commit` CAS not yet, `WorktreeLock` held. A reconciliation worker runs + `reconcile_worktree_inner` with a contention-signalling closure and must + block on the same lock (asserted: contention observed; the reconcile-done + channel stays empty for 300 ms). Releasing the coordinator lets the real + prepare + `store.commit` CAS commit X and drop the lock; reconciliation then + acquires the lock, reads `load_all_tree_roots`, and retains X + (`report.deleted == 0`, `report.local_required == 2`). Asserts X's + `refs/sce/mutation-cursor//` ref survives (`git show-ref --verify`), + and the DB proves X durable through the real flow (`cursor_tree == X`, a + `mutation_trace_events` row with `before_tree == baseline`, `after_tree == + X`). Deterministic `mpsc` channels only — no sleeps as the synchronization + mechanism. + (3) `context/cli/mutation-trace-ref-reconciliation.md` — the "Locking" + section now names both regressions and states precisely what each proves + (T04-equivalent = generic `WorktreeLock` happens-before, X made durable + directly; T07-equivalent = the same property across the real `capture → pin → + load → prepare → store.commit CAS` path via the `pub(super)` `after_load` + coordinator seam, test-only, production passes a no-op). "Testing boundary" + and the `reconcile_worktree_inner` seam note updated to match; the stale + "T04/T05 tests" / "deterministic pin→CAS lock-race regression" phrasings + (deferred to T07 by T05) are gone. + Plan AC5 / Q18 / AC16 already state the T04-vs-T07 distinction precisely + (pre-authored with this revision) — no plan-body edit was needed for the + "state precisely what T04 vs T07 prove" done-check. + - Verify (actual): + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::` + (via `nix develop -c`) — `ok. 80 passed; 0 failed` (was 79), including the + new `reconciliation_blocks_until_a_real_coordinate_cas_commits_the_pinned_tree` + and the kept T04 `reconciliation_blocks_on_the_worktree_lock_and_retains_a_pin_that_becomes_durable_under_it`. + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` + — `Finished`, no warnings. + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check` + — clean (exit 0, no diff). + - Deviations: `#[allow(clippy::too_many_lines)]` on the new test — the test is + ~150 lines (two coordinated worker threads plus DB/ref assertions), and the + file already carries the same allow on three comparably sized tests. No + smaller shape preserves the deterministic pin→CAS interleaving. + Per a user instruction during implementation, all explanatory comments added + by this task were removed — the new `coordinate_inner` seam and the new test + carry no added `//` or `///` comments. + Doc scope: only `context/cli/mutation-trace-ref-reconciliation.md` (named in + the task) was edited during execution; the `mutation-trace-runtime-coordinator.md` + seam note and the root-context annotations remain for the context-sync phase + (this plan's "Context sync" section, T07 items), matching the T06 precedent. + - Context impact: `coordinate_inner` visibility widened from module-private + (`coordinator.rs`) to `pub(super)` (`runtime`), and it and + `coordinate_protected` gained an `after_load` closure parameter — all + test-only reach; production `coordinate()`'s signature and behavior are + unchanged, no re-export, no caller outside `runtime`. No schema, migration, + protocol, marker, spec, Quint, or dependency change. Durable-context + follow-ups already enumerated in this plan's "Context sync" section (T07 + items): `context/cli/mutation-trace-runtime-coordinator.md` (note the + `pub(super)` `after_load` seam) and, in + `context/cli/mutation-trace-ref-reconciliation.md`, the already-applied T04 + reframing. Root-context files: `context/architecture.md`, + `context/glossary.md`, `context/patterns.md`, `context/overview.md`, + `context/context-map.md` — a test-only coordinator seam adds no module, + term, pattern, or architectural boundary; verify-only in context sync. + - Context synchronization: synced + +- [x] T08: `Migrate runtime/tests.rs filesystem fixtures to RAII-owned TempDir` (status:done) + - Task ID: T08 + - Scope: In — `cli/src/services/mutation_trace/runtime/tests.rs` only. + Introduce an RAII fixture (e.g. + `struct TestRuntimeRepo { _temp_dir: tempfile::TempDir, repo_root: PathBuf, + db_path: PathBuf }`, plus a linked-worktree variant if the linked tests + need a different shape) laying out `TempDir/{repo/, linked/, agent-trace.db}` + with the DB **outside** any captured worktree. Convert every existing test + in the file to the fixture; remove all manual `cleanup(..)` calls and the + `NEXT_ID` / `unique_path` / `SystemTime` / `UNIX_EPOCH` / `AtomicU64` / + `Ordering` / `cleanup` scaffolding (keep only what a specific remaining + test still needs, and say why in a comment). If `TempDir` drop is + unreliable on a supported platform while linked Git worktrees exist, + `git worktree remove` them in the fixture's `Drop` (or explicitly before it) + while keeping the outer filesystem lifecycle RAII-owned — no sleeps, no test + serialization, no global mutex, no retries. Out — new integration scenarios + (T09); production-code changes; the outcome enum (T06); the T04 reframing + (T07). + - Dependencies: T06, T07 + - Done when: no test in `runtime/tests.rs` depends on reaching a manual + `cleanup()` to avoid a stale top-level temp dir; the obsolete scaffolding is + gone; a panic in any migrated test cannot leak its temp repository; + `services::mutation_trace::runtime::tests::` passes; clippy `-D warnings` + clean; `fmt --check` clean. + - Verify: `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::tests::`; + `grep -n "fn cleanup\|unique_path\|AtomicU64\|UNIX_EPOCH\|NEXT_ID" cli/src/services/mutation_trace/runtime/tests.rs`; + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings`. + - Completed: 2026-09-01 + - Files changed: `cli/src/services/mutation_trace/runtime/tests.rs` + - Result: Introduced two RAII test-only fixtures — `TestRepo` (single worktree: + `TempDir/{repo/, agent-trace.db}`) and `LinkedTestRepo` (linked worktrees: + `TempDir/{main/, linked/, agent-trace.db}`) — mirroring the `TempDir` + fixture pattern already established in `git_snapshot.rs` and + `ref_reconciliation.rs` during T02/T03. Every one of the 12 tests in the + file (10 single-worktree, 2 linked-worktree) now constructs its repository + and DB through one of these fixtures instead of `unique_path` + + `std::env::temp_dir()`, and no test calls a manual `cleanup(..)`; the + `TempDir`'s `Drop` removes the whole tree on success, panic, or early + return. Removed the `NEXT_ID` / `unique_path` / `SystemTime` / `UNIX_EPOCH` + / `AtomicU64` / `Ordering` / `cleanup` scaffolding entirely — no remaining + test needs it. The linked-worktree fixture places both worktrees under one + `TempDir` with no explicit `git worktree remove` before drop: nothing + outlives the fixture that could observe a stale `.git/worktrees/` entry + once the whole tree is removed together, and the prior manual `cleanup()` + (a plain `remove_dir_all`) already relied on the same property for these + same two tests. No production code changed; no test behavior or assertion + changed except reopening the DB for assertions via + `RepositoryAgentTraceDb::open_for_hooks_without_migrations_at` (the + idiom already used by the majority of tests in this file) in the two tests + that previously held the `new_at`-returned handle across the whole test. + - Verify: `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::tests::` + — 12 passed, 0 failed; + `grep -n "fn cleanup\|unique_path\|AtomicU64\|UNIX_EPOCH\|NEXT_ID" cli/src/services/mutation_trace/runtime/tests.rs` + — no matches; + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` + — clean; + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check` — + clean (after running `cargo fmt` to apply formatting); + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` — 881 + passed, 0 failed (full CLI suite, confirming no other module regressed). + - Context impact: Test-only fixture reorganization confined to + `cli/src/services/mutation_trace/runtime/tests.rs`, which compiles only + under `#[cfg(test)]`. No production type, module, behavior, dependency, + schema, protocol, or public-interface change. No context doc references + this file's internal test-fixture shape, so no durable context file + requires an update for this task. + - Context synchronization: synced + +- [x] T09: `Add public/runtime reconciliation integration suite` (status:done) + - Task ID: T09 + - Scope: In — `cli/src/services/mutation_trace/runtime/tests.rs`, using the + T08 RAII fixture against real `git init` / `git worktree add` repositories + and a real repository-scoped `RepositoryAgentTraceDb` (DB in a sibling temp + dir, outside every worktree). **Test-boundary contract:** the behavior being + verified is always exercised through `reconcile_worktree` / + `reconcile_worktree_inner` (and, for normal-flow setup, `coordinate`). + Normal-behavior scenarios prefer real runtime flows end to end + (`coordinate → mutate filesystem → coordinate → reconcile_worktree`). + Lower-level `GitSnapshotService::{capture_tree, pin_tree}` calls and direct + DB (raw-SQL) seeding are permitted **only** to construct intentionally + orphaned / degraded / corrupted / otherwise-unreachable prerequisite states + that a public flow cannot reach; the operation under test in those + scenarios is still `reconcile_worktree(...)`. The suite must not claim the + entire setup uses only public APIs. Scenarios: + - active-worktree orphan deletion: `coordinate()` baseline → capture & + `GitSnapshotService::pin_tree` a tree X with **no** durable root → later + `reconcile_worktree` deletes exactly X's ref (post-crash / post-no-op + state `pin exists ∧ durable root does not`; no coordinator crash seam); + - current cursor retention without a referencing event; + - historical `before_tree` / `after_tree` retention driven through real + `coordinate()` `A→B→C→D` transitions (prefer real transitions; raw SQL + only for intentionally degraded / unreachable states); + - idempotence: first pass deletes N, second deletes 0, retained / root + counts stable; + - linked-worktree isolation: `reconcile_worktree(A)` enumerates / deletes + no `refs/sce/mutation-cursor//` ref, needs no pause in B, deletes no + tree B durably requires, including the byte-identical-tree-content case; + - cross-worktree degraded-state retention: B durably references T, B's own + `refs/sce/mutation-cursor//T` deliberately absent, A owns + `refs/sce/mutation-cursor//T`, A does not durably reference T → + `reconcile_worktree(A)` **retains** A's T pin and `git cat-file -t T` + still succeeds; + - missing local required pin: `T ∈ durable_roots(A)`, `T ∉ pinned_trees(A)` + → `MissingRequiredPins`, nothing deleted, **even if** another worktree + pins T; + - malformed / symbolic namespace ref in A's SCE namespace → fail closed, + all refs untouched; + - no DB / protocol mutation on a normal pass — capture the runtime-observable + state before the pass (`mutation_trace_worktrees` / `_events` / `_scopes` / + `_processed_events` / `_event_active_scopes` row counts; `cursor_tree` / + `revision` / `tainted` / `failure_kind` / `needs_rebaseline`; the + external-taint marker state; the readable repository schema/migration + version), run reconciliation, then assert every one is unchanged and only + Git refs differ. The source-tree guarantee that no + `cli/migrations/agent-trace-repository/` file was added or changed is + **not** a runtime assertion — it is covered by AC11's inspection step and + by `/validate`'s changed-file review, not by this integration test; + - no object reclamation: an object reachable before the pass only through a + stale SCE pin is still resolvable via `git cat-file -t` immediately after + that pin is deleted (no `git gc` / `git prune` / `git reflog expire`); + - missing checkout identity through the public entrypoint returns + `ReconciliationOutcome::SkippedNoCheckoutIdentity`, not a zero-count + `Reconciled(..)` report. + The conditional-delete *atomicity* race is proven in T02 against + `delete_pins` directly (its `delete_pins_inner` `after_preflight` seam), not + re-scheduled through the public reconciler. Out — any production-code + change; documentation edits (each task owns its own context sync); the + manual-cleanup fixture pattern. + - Dependencies: T08 + - Done when: every scenario above passes with the behavior under test driven + through `reconcile_worktree` / `coordinate` (lower-level Git/DB helpers used + only to build deliberately degraded prerequisite states), with no + production-code change; `services::mutation_trace::runtime::` and the full + CLI test suite pass; clippy `-D warnings` and `fmt --check` clean. + - Verify: `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::tests::`; + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml`; + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings`; + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check`. + - Completed: 2026-09-01 + - Files changed: `cli/src/services/mutation_trace/runtime/tests.rs` + - Result: Added the eleven integration tests the scenario list calls for, + every one driven through the public `reconcile_worktree` entrypoint (and, + for normal-flow setup, `coordinate`) against real Git repositories and a + real repository-scoped `RepositoryAgentTraceDb`, reusing the T08 + `TestRepo` / `LinkedTestRepo` fixtures: `a_pin_with_no_durable_root_is_reclaimed_by_a_later_reconciliation` + (AC2, active-worktree orphan deletion), `current_cursor_pin_survives_reconciliation_without_a_referencing_event_through_the_public_api` + (AC3), `historical_before_and_after_pins_survive_reconciliation_after_real_coordinate_transitions` + (AC4, driven through four real `coordinate()` boundaries producing a real + A→B→C→D history), `reconciliation_through_the_public_api_is_idempotent` + (AC8), `reconcile_one_linked_worktree_leaves_the_other_worktrees_pins_and_shared_objects_intact` + and `reconcile_a_retains_its_pin_when_another_worktree_durably_requires_the_same_tree` + (AC9, the second constructing byte-identical tree content independently in + both worktrees), `missing_local_required_pin_fails_closed_and_deletes_nothing_even_when_another_worktree_pins_the_tree` + (AC6), `a_malformed_namespace_ref_fails_closed_through_the_public_entrypoint` + (AC7, a symbolic ref onto the worktree's own well-formed cursor pin), + `reconciliation_makes_no_protocol_or_marker_write` (AC11, asserting every + mutation-trace table's row count and the whole `WorktreeState` byte- + identical around a pass that still deletes one real orphan ref), + `reconciliation_deletes_a_stale_ref_without_reclaiming_the_object_through_the_public_api` + (AC12), and `missing_checkout_identity_through_the_public_entrypoint_returns_skipped_outcome` + (AC14/AC15, with a panicking `open_db` closure proving it is never called). + Lower-level `GitSnapshotService::{capture_tree, pin_tree}` calls and direct + raw-SQL `seed_event` seeding are used only to construct the deliberately + orphaned / degraded / malformed prerequisite states a public flow cannot + reach; the operation under test is always `reconcile_worktree(...)` itself. + Two small test-only helpers were added (`ref_exists`, `row_count`, + mirroring the equivalent helpers already in `git_snapshot.rs` and + `agent_trace_db/repository.rs`), and the existing `run_git` helper now + returns the captured stdout (as `git_snapshot.rs`'s sibling `run` helper + already does) so a test can read back a ref's resolved value, not only + assert the command succeeded. No production code changed. + - Verify: `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::tests::` + — 23 passed, 0 failed (12 pre-existing + 11 new); + `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` — 892 + passed, 0 failed (full CLI suite; one earlier parallel run showed 5 + unrelated `services::agent_trace_export::tests::` failures that did not + reproduce in isolation or on a clean re-run, confirming pre-existing + parallel-execution flakiness unconnected to this task's file); + `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` + — clean; + `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check` + — clean (after running `cargo fmt` to apply formatting). + - Context impact: New tests only, confined to + `cli/src/services/mutation_trace/runtime/tests.rs`, which compiles only + under `#[cfg(test)]`. No production type, module, behavior, dependency, + schema, protocol, or public-interface change. `context/cli/mutation-trace-ref-reconciliation.md`'s + and `context/cli/mutation-trace-runtime-coordinator.md`'s "Testing + boundary" sections named `runtime/tests.rs`'s prior, narrower coverage and + needed updating to describe the full T09 suite; no other context file was + affected. + - Context synchronization: synced + +## Design decisions + +### Q1 — Why the existing per-worktree `WorktreeLock` is sufficient + +`coordinate()` acquires `/sce/mutation-cursor.lock` (an OS advisory +lock via `std::fs::File::try_lock`, RAII-released) **before** arming the +external-taint marker and resolving checkout identity, and holds it across the +entire pipeline: `capture + pin → recover-if-needed → prepare/commit → DB CAS +→ marker clear → return` +(`context/cli/mutation-trace-runtime-coordinator.md`). The pin is therefore +created strictly inside the lock hold, and the tree stays "possibly not yet +durable" only until the DB CAS inside that same hold. + +`reconcile_worktree` acquires the **same** lock file with the **same** +primitive (`worktree_lock::acquire_inner`, bounded by its own +`RECONCILIATION_LOCK_TIMEOUT` of 10s — a value that matches the coordinator's +private `WORKTREE_LOCK_TIMEOUT` by intent, not a shared constant) before it +lists pins, reads durable roots, or deletes anything. +OS advisory locks on one file are mutually exclusive, so the reconciler's +critical section runs entirely before `coordinate()` takes the lock or +entirely after `coordinate()` releases it — never interleaved. In the "before" +case the in-flight tree is not pinned yet, so it cannot be a deletion +candidate. In the "after" case `coordinate()` has already resolved the tree's +fate: committed (it is now a durable root and appears in the durable-root +queries → retained) or not committed (a genuine orphan, safe to delete). There +is no third state a reconciler can observe. No new lock, and no +repository-global lock, is needed. + +The retention set the pass computes is repository-wide (Q2, Q4), but that is a +broadening of a **read**, not of the **lock**. An in-flight tree in worktree +`W` can only be created while `W`'s lock is held (above); a *concurrent* +worktree `B` always creates its own `refs/sce/mutation-cursor//X` before +`B` commits `X` durably. That repository-wide read is protected from a torn +view of a concurrent commit not by any lock but by being **one SQL +statement** — a single `UNION` of `cursor_tree` / `before_tree` / +`after_tree` read through one DB snapshot (Q4, AC1), so an atomic +`cursor T → X` + `event T → X` commit on `B` cannot expose a root set that +omits `T`. So a per-worktree lock plus a single-statement repository-wide +read is sufficient — see Q2, Q4, and "The challenge interleavings". + +### Q2 — Linked worktrees: per-worktree lock, repository-wide retention set + +Linked worktrees share one object database and one default ref namespace, but +each has a distinct `git-dir` and therefore a distinct +`/sce/mutation-cursor.lock` and a distinct `WorktreeId` (checkout +id). `pin_tree` scopes every ref by the `worktree_id` path segment +(`refs/sce/mutation-cursor//`). `reconcile_worktree(A)` +still **mutates** only A's world: + +- it lists only `refs/sce/mutation-cursor//` (Q10 — `git for-each-ref` + constrained to that exact prefix); +- it deletes only refs it inventoried under A's prefix; +- it writes no row, and takes no lock other than A's. + +But the deletion *decision* cannot be made from A's rows alone. Because the +object database is shared, an A-owned ref may be the last SCE ref protecting a +tree that **only worktree B** durably requires — B's `cursor_tree`, +or a `before_tree` / `after_tree` of one of B's historical events. B's +database row does not itself hold that tree reachable to Git; it means +reconciliation must keep some SCE ref protecting it. If A +decided staleness as `actual_A − load_tree_roots(A)`, it could delete +`refs/sce/mutation-cursor//T` while B still requires T, leaving **no** SCE +ref to protect T — so a later `git gc` could reclaim the objects and B's +durable cursor/evidence tree becomes unresolvable. That violates the core rule +that a pass must never remove a ref whose target tree any durable +mutation-cursor state in the repository still requires SCE to protect. + +So A **must read** B's durable `TreeId`s — via `load_all_tree_roots()`, a +single SQL statement `UNION`-ing `cursor_tree` / `before_tree` / `after_tree` +across every worktree, read through one DB snapshot (Q4, AC1) — and remove an +A-owned ref only when its tree is in **no** worktree's durable root set: + +``` +stale_A = actual_A − load_all_tree_roots() +``` + +The corrected statement of the cross-worktree boundary: **A never mutates B's +refs or rows, but A must consider B's durable `TreeId`s when deciding whether +an A-owned ref is safe to remove from the shared Git object database.** + +A repository-global lock is still not needed. The pin→CAS race is closed by +A's own per-worktree lock (Q1); a concurrent B always creates its own pin +before committing its tree; and the repository-wide root *read* is protected +from a torn view of B's atomic `cursor T → X` + `event T → X` commit by being +**one SQL statement** over one DB snapshot (Q4, AC1), not by any lock. So a +repository-wide single-statement *read* under a per-worktree *lock* is +race-free (see "The challenge interleavings"). Object identity is +content-addressed: if A and B pinned byte-identical content they pinned the +*same* object under two refs, and deleting A's ref cannot unreach it while B's +ref names it; object reclamation is Git's job, performed only for genuinely +unreachable objects on Git's own schedule (Q12). A repository-global lock +would only add contention with live coordinator traffic on unrelated +worktrees for no safety gain. + +### Q3 — How the reconciler obtains the correct `WorktreeId` + +It does not accept one. `reconcile_worktree(repository_root, open_db)` derives +identity exactly as `coordinate()` does: `repository_root` → +`checkout::resolve_git_dir` (returns the worktree-specific git-dir, including +for a linked worktree) → `checkout::read_checkout_id(&git_dir)` → +`WorktreeId(id)`. It uses `read_checkout_id` (not `get_or_create_checkout_id`) +so a read-only maintenance pass never creates an identity as a side effect. +The safety argument rests on **derivable current identity**, not on a claim +about history: `read_checkout_id` → `Ok(None)` means reconciliation has no +current checkout identity — no readable canonical `/sce/checkout-id` +— from which it can derive a `WorktreeId` and therefore no safe worktree-owned +`refs/sce/mutation-cursor//` prefix to reconcile. It is **not** a +claim that no `WorktreeId` or ref ever existed for this checkout (those may be +the Q16 unowned namespaces), only that none can be safely derived now. + +The final planned outcome (T06 — see Q17) for that branch: + +``` +no checkout identity + ↓ +no WorktreeId / owned ref prefix can be derived + ↓ +no namespace inventoried + ↓ +no DB opened/read + ↓ +no refs mutated + ↓ +Ok(ReconciliationOutcome::SkippedNoCheckoutIdentity) +``` + +This is deliberately **not** +`Ok(Reconciled(ReconciliationReport { 0, 0, 0 }))`: the latter means a real +reconciliation pass ran (a checkout identity existed, the namespace was +inventoried, both durable-root reads happened) and there was simply no work to +do (Q6). `SkippedNoCheckoutIdentity` carries no counts because no pass ran. The +already-acquired `WorktreeLock` is RAII-released on this return and no identity +is created. `read_checkout_id` → `Err(_)` is a different thing entirely: an +unreadable or corrupt checkout id is not an absent one, so it maps to +`ReconcileError::CheckoutIdentity` rather than a skip. + +Current code truth: T03 shipped the interim behaviour +`Ok(ReconciliationReport { 0, 0, 0 })` for this branch. T06 replaces it with +`Ok(ReconciliationOutcome::SkippedNoCheckoutIdentity)`; every normative +statement in this plan describes the T06 target. + +### Q4 — The complete durable root set (per-worktree and repository-wide) + +Verified against `cli/migrations/agent-trace-repository/003_mutation_trace_protocol.sql` +and `store.rs`. The five mutation-cursor tables and their `TreeId`-typed +columns: + +| Table | `TreeId` columns | +| --- | --- | +| `mutation_trace_worktrees` | `cursor_tree` | +| `mutation_trace_events` | `before_tree`, `after_tree` | +| `mutation_trace_scopes` | none | +| `mutation_trace_processed_events` | none | +| `mutation_trace_event_active_scopes` | none | + +`AttemptState` is never persisted (transient), and `external_taint` is never +DB-authoritative — both explicitly excluded by `store.rs` and migration `003`. +So the complete durable root set for one worktree is: + +``` +{ mutation_trace_worktrees.cursor_tree(W) } + ∪ { mutation_trace_events.before_tree : row for W } + ∪ { mutation_trace_events.after_tree : row for W } +``` + +`load_tree_roots(W)` returns exactly this for one `W`, deduplicated, as a +`BTreeSet`. `load_all_tree_roots()` returns `⋃_V` of the same +expression across every worktree `V` — the same three columns +(`mutation_trace_worktrees.cursor_tree`, `mutation_trace_events.before_tree`, +`mutation_trace_events.after_tree`) with **no** `WHERE worktree_id` clause — +deduplicated. No other table is part of the durable root set in either query. + +Cursor advances that emit a `MutationEvent` record the prior cursor as that +event's `before_tree`, so it stays a root; a cursor that moved via recovery +(no event) is correctly *not* a root and its pin becomes reclaimable **once no +worktree needs that tree**. Both queries are read-only over existing columns — +no migration is needed. + +**Single-statement snapshot semantics.** Each of `load_tree_roots` and +`load_all_tree_roots` produces its complete root set from **one SQL +statement** — a `UNION` of the `cursor_tree`, `before_tree`, and `after_tree` +arms — read through **one** `query_map` call. The constituent tables are +never queried with independent `SELECT`s whose result vectors are later +unioned in Rust. This matters because a mutation-cursor commit atomically +performs, in a single DB transaction: + +``` +cursor_tree: T → X +INSERT MutationEvent { before_tree = T, after_tree = X } +``` + +If the root set were assembled from two independent `SELECT`s, one could read +`mutation_trace_events` before that transaction (T absent there) and +`mutation_trace_worktrees` after it (cursor already X), yielding a set that +omits T entirely — even though T is still a live durable root. The +one-statement read cannot do this: it observes either the pre-commit snapshot +(`cursor_tree` contains T) or the post-commit snapshot (`before_tree` +contains T, `after_tree` contains X). There is **no** snapshot in which +`cursor_tree` no longer contains T **and** `before_tree` does not yet contain +T, because the cursor update and the event insert commit together and the +query sees them through one statement. The safety does **not** rest on +"we query the cursor table first" or any ordering of separate reads — it is +structural. + +The reconciler uses the per-worktree set for the fail-closed **local +consistency** check and the repository-wide set for the **deletion safety** +check (Q7, "Core invariants"). + +**Two distinct concurrency arguments, kept separate.** This design defends +against two unrelated races, with two different mechanisms: + +1. **Git pin → DB CAS race** (same worktree): a `coordinate()` that has + pinned a tree but not yet committed it durably. Guarded by the + per-worktree `WorktreeLock` — the reconciler takes the same lock file + `coordinate()` holds across `pin → CAS → return`, so it can never observe + a pinned-but-uncommitted tree (Q1). +2. **Repository-wide durable-root read vs. a concurrent commit on another + worktree**: a torn view of the cursor/event tables while some other + worktree's `coordinate()` commits. Guarded by the **single SQL statement + snapshot** above — the whole root set comes from one coherent DB snapshot, + so an atomic `cursor T → X` + `event T → X` commit on another worktree can + never expose a mixed pre/post-commit root set (this Q4, and "The challenge + interleavings"). + +The complete model: **same worktree — `WorktreeLock` prevents seeing a +pinned-but-uncommitted tree; other worktrees — each coordinator creates its +pin before its atomic DB commit, and reconciliation reads the cursor/event +durable roots from one DB snapshot.** No repository-global lock is needed. + +### Q5 — Worktree never materialized in the DB + +The ordering is `resolve_git_dir` → **acquire `WorktreeLock`** → +`read_checkout_id` → …. So `read_checkout_id` → `Ok(None)` means: the +reconciler has already acquired `WorktreeLock(W)`, then finds no checkout +identity, then **releases the lock and returns** +`Ok(ReconciliationOutcome::SkippedNoCheckoutIdentity)` (T06 target; T03 shipped +the interim `ReconciliationReport { 0, 0, 0 }`) without creating identity and +without any DB provider call, pin inventory, or Git-ref work — reconciliation +cannot derive a current `WorktreeId`, so it has no worktree-owned ref prefix to +reconcile (Q3, Q17). The return happens *after* lock acquisition, not +before it. `read_checkout_id` → `Err(_)` ⇒ `ReconcileError::CheckoutIdentity` +(a corrupt id is not an absent id). If the checkout id exists but there is no +`mutation_trace_worktrees` row (a `coordinate()` that pinned then failed +before `initialize_worktree`, or a different Agent Trace path created the +checkout id), `load_tree_roots(W)` returns the empty set, `required_local` is +empty, `missing_local` is empty, and every pin under the prefix that is also +absent from `load_all_tree_roots()` is stale and deleted — the orphan-pin +case (Q7 note, AC2). Safe: nothing durable references those trees and the +lock guarantees nothing in-flight does. + +### Q6 — No refs for a worktree + +This is the case where a **real reconciliation pass runs and finds no work** — +distinct from the Q3/Q17 skip. A checkout identity exists, so a `WorktreeId` +and owned prefix are derived, `open_db()` is called, and both durable-root +reads execute: + +``` +checkout identity exists +actual pins = {} +required_local = {} + ↓ +Ok(ReconciliationOutcome::Reconciled(ReconciliationReport { + local_required: 0, + retained: 0, + deleted: 0, +})) +``` + +`list_pins` returns an empty vector. If `required_local` is also empty ⇒ the +zero-count `Reconciled` report above, success, idempotent. If `required_local` +is non-empty ⇒ every locally required tree is missing a pin ⇒ +`MissingRequiredPins` fail-closed (Q7). + +### Q7 — A durable root of the *target* worktree has no corresponding ref + +Fail closed. `missing_local = load_tree_roots(W) − {p.tree for p in actual_W}` +non-empty ⇒ `Err(ReconcileError::MissingRequiredPins { missing: missing_local +})`, **delete nothing**. This is the **local consistency invariant** +violation: a tree `W`'s own durable evidence still references has lost its +pin, so a `git gc` could already have reclaimed its objects. + +This check is deliberately **local**, never repository-wide. A missing pin in +some *other* worktree `B` is not a reason to abort `A`'s pass — requiring +every worktree to hold a complete pin set before `A` could reconcile would +let one worktree's degradation block maintenance everywhere, for no safety +gain. Instead: if `B` requires `T` and `A` also has a `T` pin, `A` must +**retain** `A/T` because `T ∈ load_all_tree_roots()` (the deletion safety +invariant, Q2). `A`'s otherwise-stale ref then acts as conservative +accidental backup reachability for `B`'s degraded state — reconciliation of +`A` cannot be the step that turns `B`'s degraded-but-recoverable state into +evidence loss (AC9, and the cross-worktree challenge interleaving). + +Automatic repair of a genuinely missing pin is out of scope — recreating the +ref only restores the guarantee if the underlying object still exists, which +requires separate reasoning this PR does not attempt. The first version +detects the inconsistency and stops; a later PR may add repair. + +### Q8 — Partial cleanup after one delete succeeds and a later one fails + +Cannot happen. `delete_pins` issues **one** `git update-ref --no-deref --stdin` +transaction containing every stale ref's `delete` command. `git update-ref +--stdin` applies all commands in a single ref transaction, committed +atomically at end of input; if any command fails (including a failed old-value +check) the whole transaction aborts and **no** ref is changed +(`git update-ref` documentation; T02 demonstrates this against this +repository's Git). So the outcome is binary: all stale refs deleted, or none +deleted and `Err(ReconcileError::DeleteTransaction)`. On `Err` the caller +re-runs the pass later; the operation is idempotent (Q, AC8). + +### Q9 — One-by-one vs. `git update-ref --stdin` atomic batch + +Atomic batch, for an obvious safety property: "every stale ref is deleted, or +none is, and each delete is conditioned on the exact SHA observed at inventory +time" (stale = inventoried under `W`'s prefix ∧ tree ∉ +`load_all_tree_roots()`). One-by-one conditional `git update-ref -d +` calls would +leave a half-cleaned namespace on a mid-sequence failure and force this plan +to define partial-cleanup semantics; the batch removes that question entirely +(Q8). T02 validates the exact stdin format +(`delete SP SP LF` per line, no explicit +`start`/`prepare`/`commit` needed) and the abort-on-mismatch behavior +experimentally, directly against `GitSnapshotService::delete_pins` via its +private test-only `delete_pins_inner(pins, after_preflight)` seam: pin two +valid trees `A` and `B`, pass the explicit batch `[A, B]` through preflight, +then have `after_preflight` retarget the second ref `B → C` so +`git update-ref --no-deref --stdin` is genuinely spawned with +`delete A_ref A` / `delete B_ref B`, its second expected-old-value check +fails, and neither delete is committed (`A_ref` still at `A`). That direct +test is the canonical proof for AC10 — a mutation *before* `delete_pins` +would instead be caught by the preflight and prove nothing about the Git +transaction. The public `reconcile_worktree` integration test (T09) does +**not** independently schedule an `after-inventory / before-delete` race — no +such deterministic seam exists on the public path (the `after_preflight` seam +is private to `git_snapshot.rs`) — it only asserts that a normal pass routes +its stale batch through `delete_pins` (stale refs gone afterward). + +### Q10 — Inventory from ref names, ref targets, or both + +Both, and they must agree. `list_pins` runs `git for-each-ref` over exactly the +prefix `refs/sce/mutation-cursor//` with the **NUL-separated** format +(hardened in T02): + +``` +git for-each-ref \ + --format='%(refname)%00%(objectname)%00%(objecttype)%00%(symref)' \ + refs/sce/mutation-cursor// + ↓ per line, NUL-separated: +refname objectname objecttype symref +``` + +Per line it requires: + +- `symref` is **empty** — mutation-cursor pins are **direct** refs; a symbolic + ref anywhere in the namespace is malformed and rejected, never followed + (without this a symref under `A`'s prefix could resolve through `B`'s ref and + be accepted as a normal pin); +- `objecttype == "tree"`; +- exactly one tree-SHA suffix segment after the worktree prefix (no extra path + component); +- the `refname` suffix SHA equals `objectname` (the target object SHA). + +The **target** (`objectname`) is the authoritative tree identity — it is what +Git reachability keys on and what the conditional delete checks — but requiring +agreement with the name catches a tampered or corrupted ref. A non-empty +`symref`, a name/target disagreement, a non-tree target, an extra path segment, +or an unparseable line is +`Err(PinInventoryError::MalformedRef { ref_name, reason })` — a variant +matchable separately from `PinInventoryError::Git(_)` (a `git for-each-ref` +that failed to run or exited non-zero) (Q11). `list_pins`'s signature is +therefore `Result, PinInventoryError>`, not +`anyhow::Result>`. `delete_pins` issues one +`git update-ref --no-deref --stdin` transaction (no-dereference so a `delete` +can never follow a symbolic ref out of the inventoried namespace). + +### Q11 — Malformed / unexpected refs inside the SCE namespace + +Fail closed. `refs/sce/mutation-cursor/**` is exclusively SCE-owned and every +ref in it is created only by `pin_tree` as a **direct** ref +(`git update-ref `). Anything else — a symbolic ref, a non-tree +target, a name/target mismatch, an unparseable `for-each-ref` line, an extra +path segment — means the reconciler's model of the namespace is wrong, and +cleanup must not proceed on a namespace it does not fully understand. `list_pins` returns +`Err(PinInventoryError::MalformedRef { ref_name, reason })`; +`reconcile_worktree` maps it deterministically to +`ReconcileError::MalformedPin { ref_name, reason }` and deletes nothing. The +sibling mapping is `PinInventoryError::Git(e) → ReconcileError::PinInventory(e)` +— a `git for-each-ref` execution failure is a different, separately matchable +outcome. (Ignoring malformed refs would also preserve the never-false-delete +invariant, but "abort on the unexpected" is the more defensible rule for a +destructive maintenance pass and is trivially observable in a test.) + +### Q12 — Observable error / result contract + +`reconcile_worktree` returns `Result` +(T06 target; T03 shipped the interim `Result`). Every fallible step in the algorithm owns exactly one +`ReconcileError` variant; there is no `Other` catch-all: + +| Step / situation | Result | +| --- | --- | +| successful reconciliation (checkout identity existed, a pass ran) | `Ok(ReconciliationOutcome::Reconciled(ReconciliationReport { local_required, retained, deleted }))` | +| a successful pass that found no work (identity exists, no pins, no local roots) | `Ok(ReconciliationOutcome::Reconciled(ReconciliationReport { 0, 0, 0 }))` (Q6) | +| `read_checkout_id` → `Ok(None)` (lock already held; no current checkout identity to derive a `WorktreeId` / owned ref prefix from — no namespace inventoried, no DB opened, no ref mutated, no identity created) | `Ok(ReconciliationOutcome::SkippedNoCheckoutIdentity)` (lock released on return; Q3, Q17) | +| `read_checkout_id` → `Err(_)` (corrupt / unreadable id, **not** absent) | `Err(ReconcileError::CheckoutIdentity(_))` | +| `resolve_git_dir` failure | `Err(ReconcileError::GitDir(_))` | +| `WorktreeLock` acquisition (timeout / I/O) | `Err(ReconcileError::Lock(WorktreeLockError))` | +| `open_db()` provider failure (maintenance error only — never arms the taint marker, never `CoordinateError`) | `Err(ReconcileError::AgentTraceDbUnavailable(_))` | +| `GitSnapshotService::new` failure | `Err(ReconcileError::SnapshotService(_))` | +| `git for-each-ref` execution failure (`PinInventoryError::Git`) | `Err(ReconcileError::PinInventory(_))` | +| malformed namespace ref (`PinInventoryError::MalformedRef`) | `Err(ReconcileError::MalformedPin { ref_name, reason })`, nothing deleted | +| `load_tree_roots` / `load_all_tree_roots` failure (DB query error, migration `003` absent) | `Err(ReconcileError::DurableRoots(_))` | +| a **target-worktree** durable root has no pin (local consistency invariant) | `Err(ReconcileError::MissingRequiredPins { missing: Vec })`, nothing deleted | +| `delete_pins` transaction failure (incl. a ref that changed since inventory) | `Err(ReconcileError::DeleteTransaction(_))`, nothing deleted (Q8) | + +`read_checkout_id() == Ok(None)` (⇒ `SkippedNoCheckoutIdentity`, a successful +skip), `read_checkout_id() == Err(_)` (⇒ `CheckoutIdentity`, a failure), and a +real zero-work pass (⇒ `Reconciled(ReconciliationReport { 0, 0, 0 })`, Q6) are +three distinct outcomes, never conflated. An `open_db()` failure here +is a reconciliation maintenance error and nothing more — it does **not** +become `CoordinateError::AgentTraceDbUnavailable` and does **not** arm +`ExternalTaintMarker`, because no mutation boundary is being coordinated. +Partial cleanup is not a representable outcome. Every error variant leaves the +ref namespace in a consistent state (either untouched, or — only on `Ok` — +with exactly the stale refs gone). + +### Q13 — Does reconciliation need `ExternalTaintMarker`? No. + +The external-taint marker exists so that a lost mutation-observation interval +(a DB write that could not be recorded) leaves a signal for the next +invocation to rebaseline conservatively. A reconciliation failure loses no +observation interval and casts no doubt on any committed `MutationEvent` — it +only means an obsolete ref (and the disk it holds) was not reclaimed this +time. Arming the marker on a reconciliation failure would force a spurious +conservative recovery and scope abandonment on the next `coordinate()`, +turning a storage-cleanup hiccup into lost attribution. Reconciliation +therefore never constructs, inspects, persists, or clears `ExternalTaintMarker`, +never calls `protocol::database_failure` / `protocol::taint` / +`protocol::recover`, and never writes any `mutation_trace_*` row (AC11). + +### Q14 — Does this need Quint / model updates? No. + +`spec/mutation_cursor.qnt` models the protocol state machine: +`worktrees.cursorTree`, `worktrees.revision`, scope lifecycle, +`processedEvents`, `mutationEvents`, attribution, taint, `externalTaint`, +recovery. `spec/mutation_cursor.md` states explicitly that "Git commands and +snapshot mechanics are not modeled" and that object reclamation / OS timing +are out of the model. Ref reconciliation: + +- does not choose or change attribution; +- does not advance the cursor or the revision; +- does not change any scope's status; +- does not create, modify, or delete a `MutationEvent`; +- only maintains Git ref *reachability* for trees the protocol has already + decided are durable. + +It operates entirely below the model boundary, on the imperative +snapshot-storage substrate the model abstracts away. No `spec/mutation_cursor.qnt` +change, and no Quint refinement-matrix entry, is warranted; the `mbt/` +harness and `checks.mutation-trace-quint-connect` must stay green unchanged. + +### Q15 — Future invocation point + +Not decided here — the deliverable is the safe primitive, not its schedule. A +reconciliation failure must never turn a successfully committed mutation +boundary into a failed one, so it must not be inlined into `coordinate()`'s +result path. Likely future call sites, for the harness-wiring PR to choose +among: a `Close` boundary (bounded per-scope frequency), a `Flush` boundary, +or an explicit `sce` maintenance / `sce doctor --fix` path. Recorded as +candidates only; this PR wires none of them and adds no `pub(crate)` +re-export. + +The harness gate (see "Scope: unowned checkout-identity namespaces"): a +**persistent / current worktree** harness lifecycle **whose checkout identity +stays stable** can be wired to the current per-worktree pass and be +storage-cleanup complete for this plan's scope. Any lifecycle that can +**retire, replace, lose, or recreate** checkout identities must not be treated +as storage-cleanup complete until the repository-scoped unowned-namespace +operation (Q16) exists — the per-worktree pass structurally cannot reach a +namespace whose checkout identity no current worktree derives. + +### Q16 — Unowned checkout-identity namespaces (post-T04 clarification, broadened post-T05/T06 review, future work) + +`reconcile_worktree` derives its owned prefix from the checkout id a **current +worktree still derives**. A `refs/sce/mutation-cursor//*` namespace +becomes **unowned** — permanently out of every per-worktree pass's reach — +whenever no current worktree can derive ``. Two lifecycles cause +this, and the future operation must reason about **checkout-identity +ownership**, not merely whether a worktree path was deleted: + +- **Case A — deleted linked worktree.** `git worktree remove` deletes the + worktree-specific git dir, so `/sce/checkout-id` is gone while + `refs/sce/mutation-cursor//*` survive in the shared repository namespace. +- **Case B — checkout-identity metadata loss / recreation.** A still-present + worktree loses `/sce/checkout-id` (id `A`); the next + `get_or_create_checkout_id` mints a fresh id `B`; the worktree now operates as + `B` while `refs/.../A/*` are unowned. Not normal operation — a metadata-loss / + recreation lifecycle the model must handle conservatively. + +Not solved here. Recorded as a future **repository-scoped** maintenance +operation: + +``` +enumerate refs/sce/mutation-cursor//* (git for-each-ref on the namespace) + ↓ +enumerate current worktrees → read each worktree's checkout-id → active checkout ids + ↓ +unowned ids (namespace present, no current worktree derives it) + ↓ +for each unowned namespace, for each pinned tree T: + T ∈ durable_roots(repository) → retain + T ∉ durable_roots(repository) → safe candidate for deletion +``` + +Constraints it must inherit from this plan: the repository-wide durability +invariant (`delete /T only if T ∉ durable_roots(repository)` — +`load_all_tree_roots()`), because an unowned namespace may still hold the only +SCE ref protecting historical `mutation_trace_events` trees other tooling needs; +the false-retention-over-false-deletion bias; and a hard prohibition on the +shortcut "checkout id is unowned → delete the whole namespace". It is a separate +PR: it needs a repository-global namespace scan and a cross-worktree active-id +inventory this plan deliberately does not build, and it is gated behind the same +harness-wiring work as invocation timing (Q15). This plan's per-worktree pass +remains the correct mechanism for high-frequency harness traffic against a +worktree with a **stable** checkout identity. + +**Harness gate.** Until this operation exists, any harness lifecycle that can +retire, replace, lose, or recreate checkout identities — an agent harness +creating and destroying ephemeral linked worktrees, or one exposed to +checkout-id metadata loss — must not be considered storage-cleanup complete. A +persistent / current-worktree harness whose checkout identity stays stable is +not gated by this. Recorded in Q15 and the "Scope: unowned checkout-identity +namespaces" section. + +### Q17 — Missing checkout identity: an explicit skipped outcome (post-T03 clarification) + +`read_checkout_id → Ok(None)` previously collapsed into +`ReconciliationReport { 0, 0, 0 }`, indistinguishable from a genuine pass that +inventoried the namespace and found nothing stale. Chosen shape — **Option A**, +an outcome enum wrapping the report: + +```rust +pub enum ReconciliationOutcome { + Reconciled(ReconciliationReport), + SkippedNoCheckoutIdentity, +} + +pub fn reconcile_worktree(..) -> Result +``` + +Option A over Option B (a `status` field on the report) because the two states +carry structurally different information — a skip has *no* counts to report, +not zero counts — and because `runtime` already prefers a matchable enum for +this kind of branch (`CoordinateError`, `CasResult`, `RuntimeBoundary`). The +skip stays an `Ok`, never a `ReconcileError` (a corrupt id is still `Err(CheckoutIdentity)`; +an absent one is not an error). Its meaning is exactly: no owned namespace was +inventoried, no durable-root comparison ran, no ref was deleted — and **not** a +claim that the repository holds no SCE mutation-cursor refs for a prior +identity (those may be the Q16 unowned namespaces; a `SkippedNoCheckoutIdentity` +return is itself Case B of that lifecycle mid-flight). The `WorktreeLock` is +acquired before the identity read and released on the skip return; no identity +is created or recovered. + +### Q18 — What T04 proves, and the exact pin→CAS regression (post-T04 review) + +T04's regression makes X durable **directly** in the test (a seeded +`mutation_trace_events` row under a re-taken lock), not through the production +coordinator `capture → pin → load → prepare → CAS` path. It is therefore a +proof of the **shared `WorktreeLock` happens-before edge** — reconciliation +blocks for the entire interval another owner holds the lock, and once it +proceeds it observes X among the durable roots and retains it — and must be +described that way. It is **not** an execution of the production CAS and the +plan must not claim it is. T04 stays as the generic shared-lock ordering +proof. + +**Decision: the exact production pin→CAS regression is required (T07), not +optional.** Code inspection shows `coordinator.rs` already has enough +deterministic hook plumbing that the seam is small: + +``` +coordinate() + → coordinate_inner(.., on_lock_contention, after_recovery) + → coordinate_protected(..) + → coordinate_boundary_inner(.., after_load, after_recovery) +``` + +and inside `coordinate_boundary_inner`, for an already-initialized worktree on +a `Flush` boundary: + +``` +capture tree +↓ +pin tree +↓ +initialize_worktree (idempotent no-op — worktree already materialized) +↓ +load_worktree +↓ +after_load(attempt) ← deterministic pause point: pin done, CAS not yet +↓ +prepare +↓ +DurableTransition::between +↓ +store.commit(...) ← real mutation-cursor CAS +``` + +So `after_load` is already exactly the "after `pin_tree`, before the real CAS" +pause point. T07 exposes the **smallest** `pub(super)` test seam that threads a +real `after_load`-style closure from `coordinate_inner` (currently a private +`fn`) down through `coordinate_protected` (which currently hardcodes +`|_attempt| {}`) into `coordinate_boundary_inner`. Production `coordinate()` +keeps passing no-op closures — no production behavior change, no new Git +abstraction, no runtime capability layer, no callback framework. + +The required regression (`reconciliation_blocks_until_a_real_coordinate_cas_commits_the_pinned_tree` +or a very close equivalent): + +``` +baseline coordinate() (worktree now initialized) + +coordinate worker (Flush): + capture X → pin X → load → after_load hook → PAUSE + | + +-- reconciliation worker starts → blocks on the same WorktreeLock + release hook → prepare → real store.commit CAS applies X → lock released + ↓ +reconciliation worker acquires the lock → load_all_tree_roots → retains X +``` + +Requirements: real coordinator CAS, real `WorktreeLock`, deterministic +channel/barrier synchronization (no sleeps as the sync mechanism), +reconciliation actually observes contention, X verified as a surviving SCE ref +afterward, and the DB proves X became durable through the actual coordinator +flow. No protocol/CAS-semantics change. + +Proof hierarchy: + +``` +T04 — the generic shared WorktreeLock happens-before property +T07 — the same property exercised across the actual + capture → pin → load → prepare → store CAS coordinator path +``` + +### Report shape + +`ReconciliationReport { local_required: usize, retained: usize, deleted: +usize }`. + +- `local_required` = `load_tree_roots(W).len()` — the target worktree's own + durable-root count, the left side of the local consistency invariant. +- `deleted` = the stale-pin count actually removed (inventoried under `W`'s + prefix, tree absent from `load_all_tree_roots()`). +- `retained` = `actual_W.len() − deleted`. + +`retained == local_required` is **not** an invariant and this plan no longer +claims it. Counter-example: `local_required = {A}`, +`load_all_tree_roots() = {A, B}`, `actual_W = {A, B}` ⇒ `local_required = 1`, +`deleted = 0`, `retained = 2` — `W`'s `B`-pin is retained because another +worktree durably needs `B`. + +The only report invariant, and it is scoped to a `Reconciled(report)` outcome: + +``` +For ReconciliationOutcome::Reconciled(report): + report.local_required ≤ report.retained + (the fail-closed check guarantees every locally required tree is pinned, + and W may additionally retain pins other worktrees need) + +For ReconciliationOutcome::SkippedNoCheckoutIdentity: + no report exists — no report invariant applies +``` + +It is wrong to say "on every `Ok` path `local_required ≤ retained`", because +`Ok(SkippedNoCheckoutIdentity)` contains no counts. + +A `repository_required` field was considered and rejected — computing it is a +second `len()` with no operational value a caller acts on. The +`ReconciliationOutcome` enum (T06) is where the "a real pass ran / no pass ran" +distinction lives; a future `dry_run` field would go on `ReconciliationReport` +without a breaking change. + +### The challenge interleavings + +**`pin X` → reconciler doesn't see X in DB → reconciler deletes X → DB commits +X.** Impossible by construction. `coordinate()` holds `WorktreeLock(W)` from +before `pin X` until after the DB CAS that commits X. `reconcile_worktree(W)` +acquires the *same* `WorktreeLock(W)` before it lists pins or reads roots. +Mutual exclusion on that one lock file means the reconciler's list-pins → +read-roots → delete sequence runs wholly before `coordinate()` takes the lock +(X not pinned yet → not a candidate) or wholly after `coordinate()` releases +it (X already committed → in `load_tree_roots(W)` → retained; or X never +committed → true orphan → safe to delete). The reconciler can only observe a +pinned-but-uncommitted X by holding the lock while `coordinate()` also holds +it, which cannot happen. The conditional `git update-ref -d +` in the atomic batch is a second line of defence for the +lock-assumption-violated or external-tampering case: a ref whose value moved +since inventory fails its delete and aborts the whole transaction. That +conditional-delete atomicity is proven directly against +`GitSnapshotService::delete_pins` in T02 (AC10), through the private +test-only `delete_pins_inner` `after_preflight` seam that mutates a +preflight-passed ref so the `git update-ref --no-deref --stdin` transaction +is genuinely issued and its expected-old-value check is what aborts the +batch — not by scheduling a mid-pass race through the public +`reconcile_worktree`, which has no deterministic +`after-inventory / before-delete` seam. + +**`reconcile_worktree(A)` reads global roots while `coordinate()` coordinates +a new tree X on worktree B concurrently.** A mutates only refs under +`refs/sce/mutation-cursor//`, writes no DB row at all, and takes no lock +but A's — but A **does read** B's durable `TreeId`s through +`load_all_tree_roots()`, and that read is a snapshot that may miss X. Safe +without a global lock: + +- `coordinate()` on B holds `WorktreeLock(B)` continuously from before + `pin X` through the DB CAS that commits X. So **B creates + `refs/sce/mutation-cursor//X` before X becomes durable.** +- If A's `load_all_tree_roots()` ran before B committed X, the only ref that + must protect X once X is durable is B's own `...//X`, which B + provably created first. A deleting some *unrelated* old A-owned pin cannot + make X unreachable. Any tree A itself pinned and abandoned is a true orphan + regardless of B. +- If A's read ran after B committed X, A sees X in the union and retains any + A-owned pin to X. + +So broadening the **retention-set read** to repository scope, while keeping +the **lock** per-worktree, closes the race. + +**Torn root read across an atomic `cursor T → X` + `event T → X` commit on +another worktree.** This is a *different* race from the two above and needs a +*different* mechanism. Consider: + +``` +worktree B: cursor_tree = T, B's own Git pin for T missing +worktree A: owns refs/sce/mutation-cursor//T (currently the last Git + ref protecting T) +``` + +A's `reconcile` computes `required_repository = load_all_tree_roots()`. +Concurrently, B's `coordinate()` runs its atomic DB transaction: `cursor_tree +T → X` **and** `INSERT MutationEvent { before_tree = T, after_tree = X }`. If +A assembled the root set from two independent `SELECT`s, this ordering is +possible: + +``` +A: SELECT ... FROM mutation_trace_events → T not present yet +B: atomic commit (cursor T→X, event T→X) +A: SELECT ... FROM mutation_trace_worktrees → cursor is X +⇒ A derives required_repository = {X}, missing T +⇒ A/T looks stale ⇒ A deletes A/T ⇒ T has no protecting Git ref +⇒ a later git gc reclaims T ⇒ B's durable evidence points at a missing tree +``` + +The `WorktreeLock` does **not** help here — it is per-worktree, and A and B +hold different locks. What closes this race is the **single SQL statement** +(AC1, Q4): `load_all_tree_roots()` reads `cursor_tree`, `before_tree`, and +`after_tree` in one `UNION` statement through one DB snapshot. That snapshot +is either entirely before B's commit (`cursor_tree` contains T ⇒ T retained) +or entirely after it (`before_tree` contains T ⇒ T retained). There is no +snapshot in which `cursor_tree` no longer contains T **and** `before_tree` +does not yet contain T, because B's cursor update and event insert commit +atomically and A's one statement observes them together. This is a structural +property of the one-statement read, not of any ordering of separate `SELECT`s +— "we happen to query the cursor table first" is explicitly **not** the +argument. + +**Cross-worktree degraded state: B durably references T, B's own T pin is +missing, A has a locally-stale pin to T, `reconcile(A)` runs.** +`load_tree_roots(A)` does not contain T (A does not durably reference it), so +`refs/sce/mutation-cursor//T` looks locally stale. But +`load_all_tree_roots()` **does** contain T (it is one of B's durable roots), +so `stale_A = actual_A − load_all_tree_roots()` excludes T and A **retains** +`refs/sce/mutation-cursor//T`. B's database row does not itself hold T +reachable — it is `load_all_tree_roots()` recognizing T as durably required +that causes reconciliation to retain the A-owned ref, and that retained Git +ref is what keeps T reachable; `git cat-file -t T` still resolves it. +Reconciliation of A therefore cannot +be the step that makes T unreachable — B's state stays +degraded-but-recoverable exactly as it was before the pass. This is AC9's +degraded-state regression, proven by the T03 inline test +`a_pin_another_worktree_durably_requires_is_retained` and re-exercised at the +public-entrypoint level in the T09 integration suite. + +The shared object database is otherwise safe because Git reachability comes +only from Git refs and history: deleting A's ref to a content-addressed object +cannot unreach that object while any B ref or repository history still names +it. A B durable root or B cursor is not itself a Git reachability edge — it is +a logical durability requirement that, through `load_all_tree_roots()`, makes +reconciliation retain an SCE Git ref protecting that tree, and that retained +ref is what supplies Git reachability. This PR runs no +`git gc` — Git reclaims unreachable objects itself, later, and only when they +are genuinely unreachable. + +## Open questions + +None. The durable root set is confirmed against migration `003` and `store.rs` +(Q4). The two-invariant model is settled: the fail-closed **local +consistency** check is per-worktree (`load_tree_roots(W)`), and the +**deletion safety** check uses the **repository-wide** durable root set +(`load_all_tree_roots()`) so an A-owned ref is never removed while any +worktree still durably needs its tree ("Core invariants", Q2, Q7). Each +root-set API reads its complete set from **one SQL statement** (`UNION` of +`cursor_tree` / `before_tree` / `after_tree`) over one DB snapshot, never +multiple independent `SELECT`s unioned in Rust, so a concurrent atomic +`cursor T → X` + `event T → X` commit on another worktree cannot expose a +torn root set that omits `T` (AC1, Q4, T01, "The challenge interleavings"). The +synchronization model reuses the existing per-worktree `WorktreeLock` via +`worktree_lock::acquire_inner` with its own bounded +`RECONCILIATION_LOCK_TIMEOUT` and **no** repository-global lock. Two distinct +concurrency arguments, two mechanisms: the same-worktree Git pin → DB CAS +race is closed by the per-worktree `WorktreeLock`; the repository-wide +durable-root read is protected from a torn view of another worktree's atomic +`cursor T → X` + `event T → X` commit by each root-set API executing +**exactly one SQL statement** (`UNION` of `cursor_tree` / `before_tree` / +`after_tree`) over one DB snapshot — never multiple independent `SELECT`s +unioned in Rust (Q1, Q2, Q4, AC1, T01, "The challenge interleavings"). The +pin→CAS race is closed by construction; it has two deterministic proofs — T04 +the generic `WorktreeLock` happens-before edge through the `pub(super) +reconcile_worktree_inner` seam (AC5), and T07 the same property across the real +`capture → pin → load → prepare → store CAS` coordinator path through a small +`pub(super)` `after_load` coordinator seam (Q18, AC16). The pin-inventory error +model is +the two-variant `PinInventoryError` mapped deterministically into +`ReconcileError` (Q10, Q11, Q12); `ReconcileError` has one variant per +fallible step and no `Other` catch-all (Q12); the two independent delete +defenses are proven separately in T02 (AC10) — preflight revalidation by the +direct-ref→symref test, and the expected-old-value atomic Git transaction by +the `delete_pins_inner` `after_preflight` seam test that mutates a +preflight-passed ref and asserts the issued `git update-ref --no-deref +--stdin` batch commits nothing; the batch-delete safety property is +atomic-or-nothing (Q8, Q9). Invocation timing is intentionally deferred to +the harness-wiring PR (Q15). + +This post-T03/T04-review revision adds: the unowned checkout-identity namespace +limitation and its future repository-scoped operation — identity-ownership +based, covering both a deleted linked worktree (Case A) and checkout-id metadata +loss / recreation on a still-present worktree (Case B) — (recorded only, Q16, +T05, AC13); an explicit `ReconciliationOutcome::SkippedNoCheckoutIdentity` +distinct from a zero-count `Reconciled` report, with the skip proven to touch no +DB and to leave the owned pin ref structurally identical (Q17, T06, AC14/AC15); +a precise reframing of +what T04 proves — the generic `WorktreeLock` happens-before edge, not a +production CAS execution — plus the **required** exact real-coordinator pin→CAS +regression through the existing `after_load` seam (Q18, T07, AC5/AC16); the +RAII-`TempDir` migration of `runtime/tests.rs` (T08, AC17); and the reshaped +public/runtime integration suite (T09, AC18). + +Open questions: None. The T07 seam question is resolved: `coordinator.rs` +already threads `after_load` into `coordinate_boundary_inner` and it is exactly +the "after `pin_tree`, before real CAS" pause point, so the exact regression is +required, not optional (Q18). + +## Final quality check + +1. **Synchronization strategy:** per-worktree, reusing the existing + `/sce/mutation-cursor.lock` `WorktreeLock` via + `worktree_lock::acquire_inner`, bounded by the module-owned + `RECONCILIATION_LOCK_TIMEOUT` (10s, matching the coordinator's private + `WORKTREE_LOCK_TIMEOUT` by intent, not a shared constant). + `reconcile_worktree` acquires it before any pin inventory, durable-root + read, or ref deletion, and holds it until return — the same lock file + `coordinate()` holds across `pin → CAS → return`. No new lock, **no + repository-global lock**; linked worktrees reconcile independently under + their own locks. Only the **retention-set read** is broadened to + repository scope (item 3), not the lock — and that broadened read is kept + coherent by being one SQL statement / one DB snapshot (item 4), not by a + lock. +2. **Why the pin→CAS race is impossible:** `coordinate()` holds + `WorktreeLock(W)` continuously from before `pin X` through the DB CAS that + commits X; `reconcile_worktree(W)` takes the same lock before it can + observe any pin. Mutual exclusion forces the reconciler's entire + inventory→diff→delete sequence to run either before X is pinned or after X + is either committed (⇒ a durable root ⇒ retained) or abandoned (⇒ a true + orphan ⇒ safe). The conditional atomic `git update-ref --stdin` delete is a + second-line guard against a ref value changing after inventory. A + *concurrent* worktree B needs no shared lock: B always creates its own pin + before committing its tree, and A's repository-wide root read is a single + SQL statement over one DB snapshot, so it cannot tear across B's atomic + `cursor T → X` + `event T → X` commit — A's single-statement repository-wide + read plus A's own lock is sufficient (Q2, Q4, "The challenge + interleavings"). +3. **Two invariants:** (a) **local consistency** — `durable_roots(W) ⊆ + pinned_trees(W)`; if false, fail closed and delete nothing + (`MissingRequiredPins`), a per-worktree check via `load_tree_roots(W)`. + (b) **deletion safety** — delete `W/T` only if `T ∉ durable_roots(repository)`, + the repository-wide set via `load_all_tree_roots()`, so an A-owned ref is + retained whenever any worktree still durably needs its tree. +4. **Durable root set:** `mutation_trace_worktrees.cursor_tree` ∪ + `mutation_trace_events.before_tree` ∪ `mutation_trace_events.after_tree` — + per worktree for `load_tree_roots`, unioned across all worktrees for + `load_all_tree_roots`, deduplicated. Each API produces its full set from + **one SQL statement** (a `UNION` of those three columns) through one + `query_map` call / one DB snapshot — never separate per-table `SELECT`s + unioned in Rust — so a concurrent atomic `cursor T → X` + `event T → X` + commit cannot expose a torn root set (AC1, Q4, T01). No other + mutation-cursor table stores a `TreeId`; `AttemptState` and + `external_taint` are never persisted. No migration needed. +5. **Task stack:** T01 `load_tree_roots` + `load_all_tree_roots` read-only + store queries (one SQL statement each — `SELECT_TREE_ROOTS_BY_WORKTREE_SQL` + / `SELECT_ALL_TREE_ROOTS_SQL`, one `query_map` per API — plus a + state-transition retention test and, separately, the deterministic + single-statement enforcement regression that asserts one + `load_*_tree_roots` call issues exactly one read statement via the + `#[cfg(test)]` `count_read_statements` seam) → T02 typed + `GitSnapshotService::list_pins` + (`Result, PinInventoryError>`) + conditional atomic + `delete_pins` (canonical AC10 proof) → T03 `runtime/ref_reconciliation.rs` + per-worktree pass under `WorktreeLock` (`RECONCILIATION_LOCK_TIMEOUT`, + `pub(super) reconcile_worktree_inner` seam, complete `ReconcileError` + contract, local fail-closed check + repository-wide deletion set) → T04 + deterministic `WorktreeLock` happens-before regression through the + `pub(super)` seam. The post-T03/T04-review revision then adds: T05 record the + unowned checkout-identity namespace limitation (Case A deleted worktree + + Case B checkout-id metadata loss/recreation) + future repository-scoped + operation (docs only, Q16, AC13) → T06 + `ReconciliationOutcome::SkippedNoCheckoutIdentity` distinct from a zero-count + `Reconciled` report (Q17, AC14/AC15) → T07 + reframe what T04 proves and add the **required** exact real-coordinator + pin→CAS regression through the `after_load` seam (Q18, AC5/AC16) → T08 + migrate `runtime/tests.rs` fixtures to RAII `tempfile::TempDir` (AC17) → T09 + the public/runtime integration suite (retained-root / orphan-as-`pin without + root` / missing-pin / idempotence / linked-worktree / cross-worktree + degraded-state retention / no-write / no-object-GC / skipped-identity, AC18). +6. **`ReconciliationReport` / `ReconciliationOutcome`:** the report stays + `{ local_required: usize, retained: usize, deleted: usize }` with + `retained == local_required` **not** an invariant — for + `Reconciled(report)` the only relation is + `report.local_required ≤ report.retained`, and `SkippedNoCheckoutIdentity` + carries no report so no report invariant applies; T06 wraps the report in + `ReconciliationOutcome::Reconciled(..)` and adds the sibling + `SkippedNoCheckoutIdentity` for the missing-checkout-identity path. +7. **Acceptance-criteria count:** 18 (AC1–AC12 from the original stack; AC13 + unowned checkout-identity lifecycle recorded; AC14/AC15 observable skipped outcome; + AC16 T04 framing + the **required** exact real-coordinator pin→CAS + regression; AC17 RAII fixtures; AC18 the public/runtime integration suite). +8. **Unresolved design questions that should block implementation:** None. The + final missing-checkout-identity contract is `SkippedNoCheckoutIdentity` + (Q3/Q5/Q12/Q17); the exact pin→CAS regression is required via the existing + `after_load` seam (Q18); no open questions remain. + +## Validation Report + +**Status:** validated +**Date:** 2026-09-01 + +### Commands run + +- `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::` -> exit 0 (262 passed, 0 failed, 630 filtered out) +- `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` -> exit 0 (892 passed, 0 failed) +- `./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` -> exit 0 (clean, no warnings) +- `./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml -- --check` -> exit 0 (no formatting diff) +- `nix flake check` -> exit 0 (`checks.cli-tests`, `checks.cli-clippy`, `checks.cli-fmt`, `checks.mutation-trace-quint-connect` all passed) +- `nix run .#pkl-check-generated` -> exit 0 (ephemeral Pkl generation passed: 141 files, inventory sha256 7dc810bf…) +- `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::runtime::` -> exit 0 (91 passed, 0 failed) — AC-named runtime/ref_reconciliation/git_snapshot tests +- `./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml services::mutation_trace::store::tests::load_` -> exit 0 (24 passed, 0 failed) — AC1 root-query tests +- `ls cli/migrations/agent-trace-repository/` -> `001_repository_schema.sql`, `002_repository_source_instance_id.sql`, `003_mutation_trace_protocol.sql` only (AC11 inspection) +- `grep -nE "fn cleanup|unique_path|AtomicU64|UNIX_EPOCH|NEXT_ID|SystemTime|Ordering" cli/src/services/mutation_trace/runtime/tests.rs` -> no matches (AC17 inspection) +- `git diff main -- cli/src/services/mutation_trace/` -> `list_pins` still constrains `git for-each-ref` to the single `refs/sce/mutation-cursor//` prefix; no `git worktree list` / repository-wide `refs/sce/**` enumeration or repo-global deletion added (AC13 inspection) + +### Success-criteria verification + +- [x] AC1: Store exposes two exact durable-root reads, each a single `UNION` SQL statement -> `load_tree_roots_returns_cursor_and_every_event_tree_deduplicated`, `load_tree_roots_excludes_other_worktrees_trees`, `load_tree_roots_is_empty_for_an_unmaterialized_worktree`, `load_tree_roots_remains_worktree_scoped`, `load_all_tree_roots_returns_every_worktree_cursor_and_event_tree_deduplicated`, `load_all_tree_roots_deduplicates_a_tree_shared_by_multiple_worktrees`, `load_all_tree_roots_is_empty_for_an_empty_repository`, `load_all_tree_roots_retains_previous_cursor_after_atomic_cursor_advance`, `load_all_tree_roots_reads_every_durable_root_in_one_sql_statement`, `load_tree_roots_reads_every_durable_root_in_one_sql_statement` all pass +- [x] AC2: Orphan/unreferenced pin deleted -> `ref_reconciliation::tests::orphan_pin_with_a_worktree_row_is_deleted`, `orphan_pin_with_no_worktree_row_is_deleted`, `runtime::tests::a_pin_with_no_durable_root_is_reclaimed_by_a_later_reconciliation` pass +- [x] AC3: Current cursor pin retained without a referencing event -> `ref_reconciliation::tests::current_cursor_pin_is_retained_without_a_referencing_event` passes +- [x] AC4: Historical `before_tree`/`after_tree` pins retained after cursor advances -> `ref_reconciliation::tests::historical_event_before_and_after_pins_are_retained_after_the_cursor_advances` passes +- [x] AC5: Reconciliation blocks on the `WorktreeLock` and retains a pin made durable under it -> `runtime::tests::reconciliation_blocks_on_the_worktree_lock_and_retains_a_pin_that_becomes_durable_under_it` passes +- [x] AC6: Missing local required pin fails closed naming the tree, deletes zero -> `ref_reconciliation::tests::a_missing_required_pin_fails_closed_and_deletes_nothing` passes +- [x] AC7: Malformed/symbolic/mismatched namespace ref -> `MalformedRef`/`MalformedPin`, deletes nothing -> `git_snapshot::tests::list_pins_rejects_a_ref_whose_target_is_not_a_tree`, `list_pins_rejects_a_ref_whose_name_disagrees_with_its_target`, `list_pins_rejects_a_symbolic_ref_inside_the_mutation_cursor_namespace`, `ref_reconciliation::tests::a_malformed_namespace_ref_fails_closed_and_deletes_nothing` pass +- [x] AC8: Idempotent (N then 0, stable counts) -> `ref_reconciliation::tests::reconciliation_is_idempotent` passes +- [x] AC9: Linked-worktree isolation + cross-worktree degraded-state retention -> `runtime::tests::reconcile_one_linked_worktree_leaves_the_other_worktrees_pins_and_shared_objects_intact`, `reconcile_a_retains_its_pin_when_another_worktree_durably_requires_the_same_tree` pass +- [x] AC10: Atomic `git update-ref --no-deref --stdin` with preflight revalidation + expected-old-value check -> `git_snapshot::tests::delete_pins_atomically_aborts_when_a_ref_changes_after_preflight`, `delete_pins_refuses_to_act_when_an_inventoried_direct_ref_became_a_symbolic_ref` pass +- [x] AC11: No protocol/durability write, no marker -> `runtime::tests::reconciliation_makes_no_protocol_or_marker_write` passes; inspection: `cli/migrations/agent-trace-repository/` holds only `001`/`002`/`003` +- [x] AC12: No `git gc`/`git prune`; object still resolvable immediately after ref delete -> `ref_reconciliation::tests::reconciliation_deletes_refs_without_reclaiming_objects` passes +- [x] AC13: Identity-based unsupported-lifecycle framing (Case A deleted worktree, Case B checkout-id metadata loss/recreation) + future repository-scoped operation recorded -> plan "Scope: unowned checkout-identity namespaces" section + Q16; `context/cli/mutation-trace-ref-reconciliation.md` and `context/cli/mutation-trace-runtime-coordinator.md` both carry the identity-ownership framing and both cases; `git diff main` shows `list_pins` still single-prefix, no repo-wide enumeration/deletion added +- [x] AC14: `ReconciliationOutcome` distinguishes `Reconciled(report{deleted:0})` from `SkippedNoCheckoutIdentity`, skip is not `Err`, lock released, no identity created -> `ref_reconciliation::tests::no_checkout_identity_returns_a_distinct_skipped_outcome` passes +- [x] AC15: Missing-checkout-identity path never calls `open_db`, no inventory, every namespace ref structurally identical -> `ref_reconciliation::tests::a_missing_checkout_identity_skip_touches_no_db_and_no_ref` passes +- [x] AC16: T04 documented as the generic `WorktreeLock` happens-before proof (not the production CAS); exact real-coordinator pin→CAS regression exists -> `runtime::tests::reconciliation_blocks_until_a_real_coordinate_cas_commits_the_pinned_tree` passes; AC5/Q18/`context/cli/mutation-trace-ref-reconciliation.md` wording confirmed; clippy clean (seam is `#[cfg(test)]`/`pub(super)`) +- [x] AC17: All `runtime/tests.rs` fixtures RAII-owned by `tempfile::TempDir`; `cleanup`/`unique_path`/`AtomicU64`/`UNIX_EPOCH`/`NEXT_ID`/`SystemTime`/`Ordering` scaffolding gone -> grep returns nothing; `services::mutation_trace::runtime::tests::` suite passes +- [x] AC18: T09 integration suite exercises every clarified contract through `reconcile_worktree`/`coordinate` against real Git + real repository DB -> full `services::mutation_trace::runtime::tests::` set passes; whole-suite `nix flake check` green + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- Reconciliation is not yet wired to any harness, hook, or command (explicitly out of scope); `reconcile_worktree` stays `runtime`-private and unreachable from outside until a follow-up harness-wiring PR (Q15). +- Repository-scoped unowned checkout-identity namespace reconciliation (Q16) is recorded as future work only; any harness lifecycle that can retire, replace, lose, or recreate checkout identities is not storage-cleanup complete until that operation exists. +- This plan builds only the reclamation half of the storage lifecycle; retained historical `mutation_trace_events` trees stay pinned indefinitely until a separate future retention/compaction lifecycle exists. diff --git a/context/plans/mutation-cursor-runtime-coordinator.md b/context/plans/mutation-cursor-runtime-coordinator.md index e359cb0f..f1bd40e1 100644 --- a/context/plans/mutation-cursor-runtime-coordinator.md +++ b/context/plans/mutation-cursor-runtime-coordinator.md @@ -46,11 +46,14 @@ construction and could miss a worktree row another caller materializes concurrently; the decision now always comes from a fresh read taken *after* the failure. Third, this plan corrects its own earlier claim that create-only ref pinning produces "bounded" storage growth — growth is -unbounded over the repository's lifetime without a reconciliation pass, and -the Follow-up PR section now sequences that reconciliation pass and the -still-deferred filesystem external-taint marker as required runtime -completion work *before* any harness adapter may become a production -consumer of this coordinator. +unbounded over the repository's lifetime, and the Follow-up PR section now +sequences a reconciliation pass and the still-deferred filesystem +external-taint marker as required runtime completion work *before* any +harness adapter may become a production consumer of this coordinator. That +reconciliation pass reclaims only orphan/crash refs left by interrupted +`coordinate()` executions and keeps them from accumulating under harness +traffic; it is not a bound on storage under a retained mutation history +(which needs separate future retention/compaction work). A third revision (further PR #244 review) corrects two remaining Git-level issues in the snapshot mechanism itself, with the surrounding architecture @@ -1542,7 +1545,7 @@ two invocations that happen to observe identical content) is a harmless, idempotent `git update-ref` — verified experimentally. **Pin lifecycle: create-only in this PR — and that growth is unbounded, not -bounded, until reconciliation exists.** The coordinator (T04) calls +bounded.** The coordinator (T04) calls `pin_tree(worktree_id, observed_tree)` exactly once per invocation, immediately after `capture_tree` succeeds and *before* any DB operation — whether or not that invocation's boundary ultimately commits, is rejected, @@ -1574,6 +1577,20 @@ why the Follow-up PR section below sequences a reconciliation pass as required runtime-completion work *before* that wiring, not as an optional later enhancement. +The follow-up reconciliation pass narrows this but does **not** by itself +make growth bounded: it reclaims only orphan/unreferenced pins — crash +artifacts, failed/no-op transition artifacts, uncommitted CAS observations — +whose tree is in no durable root anywhere in the repository. Every pin for a +tree that is still a current or historical durable root (worktree +`cursor_tree`, or a historical `mutation_trace_events` +`before_tree`/`after_tree`) is retained, and those event rows are retained +indefinitely, so a repository with a long retained mutation history still +holds one pin per distinct historical tree. Actually bounding that requires a +separate future retention/compaction lifecycle — recorded as future work in +`context/plans/mutation-cursor-ref-reconciliation.md`, not designed here. +Reconciliation's runtime-completion gate is specifically about keeping +orphan/crash refs from *accumulating* under harness traffic. + This directly answers "when can refs safely be deleted": not from inside this PR's per-invocation coordinator at all; only from a pass with visibility into the full set of durable references a ref might still be @@ -2514,7 +2531,14 @@ for a direct, itemized accounting. "bounded, linear" description was imprecise and is corrected above. Growth is linear *in hook-invocation volume*, which is itself unbounded over a repository's life; nothing in this PR's own scope ever removes a - pin. + pin. Note that the follow-up `mutation-cursor-ref-reconciliation` pass + does **not** by itself make this bounded either: it reclaims only + orphan/unreferenced pins (crash artifacts, failed/no-op transitions, + uncommitted CAS observations). Every pin for a current or historical + durable root — the retained-roots contract below — stays pinned, and + historical `mutation_trace_events` rows are retained indefinitely, so + bounding storage under a retained history needs a separate future + retention/compaction lifecycle, not this reconciliation pass. 18. **What exact PR must add ref reconciliation, and what exact PR must add filesystem external-taint durability?** `mutation-cursor-ref-reconciliation` and `mutation-cursor-external-taint` respectively — see "Runtime @@ -2620,14 +2644,20 @@ Design decisions) rather than leaving it unstated; this follow-up is where it closes. **Step 3 — ref reconciliation (why it cannot wait):** see "Pin lifecycle: -create-only in this PR — and that growth is unbounded, not bounded, until -reconciliation exists" above for the corrected growth analysis and the +create-only in this PR" above for the corrected growth analysis and the retained-roots contract this follow-up must implement (worktree `cursor_tree` and every historical `mutation_trace_events` `before_tree`/`after_tree`), plus the concurrency-safety requirement any implementation must satisfy (never delete a ref for a tree that could still become durable — see that section for the two candidate strategies this -plan leaves for the follow-up to choose between). +plan leaves for the follow-up to choose between). Its purpose is to keep +**orphan/crash snapshot refs** — pins left by interrupted `coordinate()` +executions — from accumulating once a harness drives high-frequency traffic; +it is *not* a claim that reconciliation bounds mutation-history storage under +normal successful usage. Pins for retained current/historical durable roots +stay pinned, and bounding that requires a separate future +retention/compaction lifecycle (`mutation-cursor-ref-reconciliation` records +this as future work). ### Step 4: harness boundary → runtime coordinator → committed `MutationEvent` → `diff_trees(before, after)` → existing Agent Trace `diff_traces` evidence @@ -2719,11 +2749,16 @@ one coordinator invocation path. ### Residual risks -- Ref-pin storage growth is unbounded until the deferred - `refs/sce/mutation-cursor/**` reconciliation pass ships, and the DB-unavailable - case leaves no filesystem external-taint marker. Both are documented, - intentionally deferred, and sequenced (`mutation-cursor-external-taint`, +- Ref-pin storage growth is unbounded: create-only pinning leaves one + permanent ref per distinct observed tree, and the DB-unavailable case leaves + no filesystem external-taint marker. Both are documented, intentionally + deferred, and sequenced (`mutation-cursor-external-taint`, `mutation-cursor-ref-reconciliation`) as required runtime-completion work before any harness adapter becomes a production consumer of `coordinate()`. + The deferred `refs/sce/mutation-cursor/**` reconciliation pass reclaims only + orphan/unreferenced pins (crash and interrupted-`coordinate()` artifacts) — + it keeps those from accumulating under harness traffic but does not by itself + bound storage under a retained mutation history, which needs a separate + future retention/compaction lifecycle. - The coordinator is standalone and not wired into any hook, CLI command, or `diff_traces` — no production caller exercises it yet. diff --git a/context/sce/shared-turso-db.md b/context/sce/shared-turso-db.md index 16eab0c5..c0e00760 100644 --- a/context/sce/shared-turso-db.md +++ b/context/sce/shared-turso-db.md @@ -23,6 +23,7 @@ - `migration_metadata_problems(&self) -> Result>`: non-mutating readiness check that queries `__sce_migrations` metadata and compares applied migration IDs against `M::migrations()`; returns a list of problems (missing metadata table, incomplete applied migrations, unexpected extra migrations) or an empty list when the schema is ready - `ensure_schema_ready(&self, setup_guidance: &str) -> Result<()>`: non-mutating hook-readiness gate that calls `migration_metadata_problems()` and bails with a formatted error including `M::db_name()` and the caller-provided guidance string when problems are found; returns `Ok(())` when the schema is ready - `passive_checkpoint(&self) -> Result<()>`: runs `PRAGMA wal_checkpoint(PASSIVE)` through the same query/runtime/retry path as `execute()`/`query()` (config-driven query retry, `block_on_isolated`), draining the checkpoint result row without exposing its busy/log/checkpointed statistics. PASSIVE checkpoints only what is currently safe to move from the WAL into the main database file and never blocks on active readers or writers, so it does not guarantee WAL truncation; safe to call repeatedly; not a durability boundary on its own. Routine maintenance only, not exposed on `EncryptedTursoDb`. `sce hooks post-commit` is the sole current caller: it runs `RepositoryAgentTraceDb::passive_checkpoint()` exactly once after post-commit Agent Trace persistence succeeds; a failing checkpoint is logged as a warning and never fails the hook or affects already-persisted data (see [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md)). +- `count_read_statements(body) -> (T, usize)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports how many `TursoDb` read statements (`query`/`query_values`/`query_map`) it issued on the current thread, each counted once in its synchronous prelude before the retry wrapper. A deterministic seam for tests that must prove an operation reads from a single DB snapshot — one statement — rather than several independent `SELECT`s a concurrent commit could tear across (first used by `MutationTraceStore::load_tree_roots` / `load_all_tree_roots`). Not compiled into production builds. - `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same structural shape (connection, runtime bridge, and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()` (environment variable `SCE_AUTH_DB_ENCRYPTION_KEY` with OS credential-store fallback), enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, wraps that open/connect block in the same connection-open retry policy resolved from `policies.database_retry..connection_open`, and runs embedded migrations after connect. - `EncryptedTursoDb` exposes the same public synchronous `execute()`, `query()`, `query_map()`, and `run_migrations()` methods; operation methods use the same config-driven query retry policy as `TursoDb`. - `TursoConnectionCore` is internal to `cli/src/services/db/mod.rs` and owns the shared Turso connection plus tokio current-thread runtime bridging used by the public adapter methods; generic embedded migration execution with per-database `__sce_migrations` metadata is delegated to `run_embedded_migrations` helpers; encryption vs unencrypted behavior remains constructor-only at the public adapter layer. diff --git a/spec/mutation_cursor.md b/spec/mutation_cursor.md index 67a1e6c4..7def4f11 100644 --- a/spec/mutation_cursor.md +++ b/spec/mutation_cursor.md @@ -47,6 +47,14 @@ externalTaint: Set[WorktreeId] `externalTaint` is the abstract external durability boundary: conceptually, the filesystem `TAINTED` marker that can survive an unavailable database. It is not a database row and does not model marker paths or filesystem syscalls. Its concrete refinement in the CLI is the worktree-local marker file `/sce/mutation-cursor-tainted`, armed write-ahead at the start of the protected runtime section — before Agent Trace DB acquisition — and cleared only after a proven durable completion; it becomes protocol `externalTaint` (overlaid onto `databaseFailure` recovery) only when a later invocation inherits it. If that trailing clear fails *after* the boundary has committed durably, the CLI keeps the marker armed and returns the committed outcome inside a distinct error rather than reporting a boundary failure, so the next invocation still recovers conservatively. See `context/cli/mutation-trace-external-taint.md`. +The SCE-owned Git snapshot refs that protect durable cursor/evidence trees +(`refs/sce/mutation-cursor//`) are **never modeled**. +They are reclaimed by an imperative per-worktree maintenance pass that deletes +only a ref whose tree is a durable root of no worktree in the repository; that +pass deletes **only SCE's own refs, never Git objects directly**, and Git +performs object garbage collection itself on its normal schedule. See +`context/cli/mutation-trace-ref-reconciliation.md`. + Thus the model does **not** perform this contradictory transition: ```text