From 06eaa5f3a2469783e3375f2631c76e11ad2bad98 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sun, 23 Aug 2026 07:42:14 +0530 Subject: [PATCH 1/3] feat(elision): invariant-bearing truncation markers; bump tok to #82 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the tok submodule to a1d1863f (tok#82: verified-fact elision summaries) and wires it into every tool-output truncation site. Hawk's markers previously said only '... (truncated)'. Measured agent behavior shows bare-count markers cause 11-97 retrieval-call storms, because the reader cannot tell whether the answer was in the dropped portion. Every marker now carries facts VERIFIED across the elided units and nothing else: - JSON tool output: '{sku-114..159} N records elided: status=shipped x46, range id=...' — field constants, enumerations summing to the total, numeric ranges, distinct-count coverage, dense-run upgrades. Structural cuts now splice at TOP-LEVEL commas (depth-aware scan) so kept records stay complete and the dropped tail parses as whole records; mid-record raw cuts still fail closed to the bare marker. - Log-shaped output: level distribution ('N lines elided: info x6'). - Prose/unknown: unchanged bare marker. Fail-closed preserved throughout: anything unparseable or under the 3-unit threshold truncates exactly as before. New elision.go exposes the shared notice builder; internal/token facades the two new tok primitives. Full engine suite green. --- external/tok | 2 +- internal/engine/elision.go | 77 ++++++++++++++++++++++++++++ internal/engine/elision_test.go | 74 ++++++++++++++++++++++++++ internal/engine/token/tok_facade.go | 12 ++++- internal/engine/tool_output_spill.go | 9 +++- internal/engine/tool_service.go | 61 ++++++++++++++++++---- internal/token/tok.go | 11 ++++ 7 files changed, 232 insertions(+), 14 deletions(-) create mode 100644 internal/engine/elision.go create mode 100644 internal/engine/elision_test.go diff --git a/external/tok b/external/tok index 643b6675..a1d1863f 160000 --- a/external/tok +++ b/external/tok @@ -1 +1 @@ -Subproject commit 643b6675ebc75b32e448fbebcd3caba92b6d7583 +Subproject commit a1d1863f360d42a9eca34572d512e26c90c8bc22 diff --git a/internal/engine/elision.go b/internal/engine/elision.go new file mode 100644 index 00000000..11470af4 --- /dev/null +++ b/internal/engine/elision.go @@ -0,0 +1,77 @@ +package engine + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/GrayCodeAI/hawk/internal/engine/token" +) + +// elisionNotice computes a verified-facts suffix for a truncation marker from +// the content being dropped. Facts state only what holds across every elided +// unit (see tok invariants): constants, exact enumerations, numeric ranges, +// distinct-count coverage. Anything uncertain is omitted — a bare count is +// better than a wrong fact, and a wrong fact reads as complete. +// +// JSON arrays of records get field-level facts; log-shaped text gets level +// distribution; anything else falls back to the line count. +func elisionNotice(dropped string) string { + trimmed := strings.TrimSpace(dropped) + if trimmed == "" { + return "" + } + if strings.HasPrefix(trimmed, "[") { + return jsonRecordsFacts(trimmed) + } + // A structural cut inside a JSON array leaves a fragment of whole records + // separated by commas, optionally ending with the array's closing bracket + // ("{…},{…}]" or "{…},{…},"). Strip both artifacts, normalize to an + // array, and retry once; anything unparseable still falls back. + if strings.HasPrefix(trimmed, "{") { + frag := strings.TrimRight(trimmed, " \t\r\n") + frag = strings.TrimSuffix(frag, "]") + frag = strings.TrimRight(frag, ", \t\r\n") + if facts := jsonRecordsFacts("[" + frag + "]"); facts != "" { + return facts + } + } + lines := splitNonEmptyLines(trimmed) + if len(lines) >= 3 { + if facts := token.LogInvariants(lines); facts != "" { + return fmt.Sprintf("%d lines elided: %s", len(lines), facts) + } + return fmt.Sprintf("%d lines elided", len(lines)) + } + return "" +} + +func jsonRecordsFacts(arrayText string) string { + var records []json.RawMessage + if json.Unmarshal([]byte(arrayText), &records) != nil || len(records) == 0 { + return "" + } + if facts := token.JSONInvariants(records); facts != "" { + return fmt.Sprintf("%d records elided: %s", len(records), facts) + } + return fmt.Sprintf("%d records elided", len(records)) +} + +func splitNonEmptyLines(s string) []string { + raw := strings.Split(s, "\n") + out := make([]string, 0, len(raw)) + for _, l := range raw { + if t := strings.TrimSpace(l); t != "" { + out = append(out, t) + } + } + return out +} + +// appendElisionMarker attaches an invariant-bearing marker to kept output. +func appendElisionMarker(kept, dropped string) string { + if n := elisionNotice(dropped); n != "" { + return kept + "\n... [" + n + "]" + } + return kept + "\n... (truncated)" +} diff --git a/internal/engine/elision_test.go b/internal/engine/elision_test.go new file mode 100644 index 00000000..49db83fe --- /dev/null +++ b/internal/engine/elision_test.go @@ -0,0 +1,74 @@ +package engine + +import ( + "strconv" + "strings" + "testing" +) + +func TestElisionNoticeJSONRecords(t *testing.T) { + var items []string + for i := 0; i < 10; i++ { + items = append(items, `{"order_id":"ord-`+strconv.Itoa(i)+`","status":"fulfilled"}`) + } + notice := elisionNotice("[" + strings.Join(items, ",") + "]") + if !strings.Contains(notice, "records elided") { + t.Fatalf("notice = %q", notice) + } + if !strings.Contains(notice, "status=fulfilled×") && !strings.Contains(notice, "distinct") { + t.Fatalf("no verified facts: %q", notice) + } +} + +func TestElisionNoticeLogLines(t *testing.T) { + var lines []string + for i := 0; i < 6; i++ { + lines = append(lines, "2026-08-22T10:00:0"+strconv.Itoa(i)+"Z INFO tick") + } + notice := elisionNotice(strings.Join(lines, "\n")) + if !strings.Contains(notice, "lines elided") || !strings.Contains(notice, "info×6") { + t.Fatalf("notice = %q", notice) + } +} + +func TestElisionNoticeProseFallback(t *testing.T) { + notice := elisionNotice("just two\nshort lines") + if notice != "" { + t.Fatalf("tiny prose should yield no notice, got %q", notice) + } +} + +func TestAppendElisionMarker(t *testing.T) { + var items []string + for i := 0; i < 12; i++ { + items = append(items, `{"id":"`+strconv.Itoa(i)+`","state":"ok"}`) + } + dropped := "[" + strings.Join(items[3:], ",") + "]" + out := appendElisionMarker(`[{"id":"0","state":"ok"},{"id":"1","state":"ok"},{"id":"2","state":"ok"}]`, dropped) + if !strings.Contains(out, "_tok") == false && !strings.Contains(out, "records elided") { + t.Fatalf("marker missing facts: %q", out) + } + // kept content preserved verbatim before marker + if !strings.HasPrefix(out, `[{"id":"0","state":"ok"}`) { + t.Fatalf("kept content altered: %q", out) + } +} + +func TestTruncateToolOutputCarriesInvariants(t *testing.T) { + var items []string + for i := 100; i < 160; i++ { + items = append(items, `{"sku":"sku-`+strconv.Itoa(i)+`","status":"shipped"}`) + } + output := "[" + strings.Join(items, ",") + "]" + // The structural cutter keeps whole records, so the dropped tail parses + // as a clean record set and invariants can be computed. A raw byte cut + // landing mid-record correctly falls back to the bare marker. + got := truncateOutputStructurally(output, 500) + if !strings.Contains(got, "records elided") { + t.Fatalf("got %q", got) + } + // The kept prefix must remain valid JSON prefix content (starts with '['). + if !strings.HasPrefix(strings.TrimSpace(got), "[") { + t.Fatal("lost array opening") + } +} diff --git a/internal/engine/token/tok_facade.go b/internal/engine/token/tok_facade.go index f5c95bcb..91d72377 100644 --- a/internal/engine/token/tok_facade.go +++ b/internal/engine/token/tok_facade.go @@ -1,6 +1,10 @@ package token -import hawktoken "github.com/GrayCodeAI/hawk/internal/token" +import ( + "encoding/json" + + hawktoken "github.com/GrayCodeAI/hawk/internal/token" +) // Stats is the compression result consumed by Hawk's runtime observations. // The alias preserves the external tok schema while keeping Tok imports inside @@ -41,3 +45,9 @@ func BuildRuntimeGraph(input RuntimeGraphInput) (*RuntimeGraphExport, error) { func Compress(text string, budget int) (string, Stats) { return hawktoken.Compress(text, budget) } + +// JSONInvariants renders verified-fact summaries for elided JSON records. +func JSONInvariants(dropped []json.RawMessage) string { return hawktoken.JSONInvariants(dropped) } + +// LogInvariants renders the level distribution of elided log lines. +func LogInvariants(lines []string) string { return hawktoken.LogInvariants(lines) } diff --git a/internal/engine/tool_output_spill.go b/internal/engine/tool_output_spill.go index a38e4498..7d43e0c3 100644 --- a/internal/engine/tool_output_spill.go +++ b/internal/engine/tool_output_spill.go @@ -51,7 +51,12 @@ func maybeSpillToolOutput(output, toolName, toolID string) string { } preview := output if len(preview) > toolOutputSpillPreview { - preview = preview[:toolOutputSpillPreview] + "\n... (see file for full output)" + preview = preview[:toolOutputSpillPreview] + if n := elisionNotice(output[toolOutputSpillPreview:]); n != "" { + preview += "\n... [" + n + "; full output in file]" + } else { + preview += "\n... (see file for full output)" + } } return fmt.Sprintf( "Output saved to %s (%d bytes).\n\nPreview:\n%s\n\nUse Read (offset/limit) or Grep on this path to inspect the rest.", @@ -63,5 +68,5 @@ func truncateToolOutput(output string, max int) string { if len(output) <= max { return output } - return output[:max] + "\n... (truncated)" + return appendElisionMarker(output[:max], output[max:]) } diff --git a/internal/engine/tool_service.go b/internal/engine/tool_service.go index 71f63473..b32b9ba5 100644 --- a/internal/engine/tool_service.go +++ b/internal/engine/tool_service.go @@ -591,27 +591,68 @@ func (s *ToolService) NormalizeOutput(output, canonicalTool, toolID string, cont // truncateOutputStructurally trims oversized tool output at a structural // boundary instead of a raw byte cut, so JSON-ish results keep whole lines -// (or a valid splice point) rather than being chopped mid-object (Phase 3). +// (or whole array elements) rather than being chopped mid-object (Phase 3). func truncateOutputStructurally(output string, maxChars int) string { trimmed := strings.TrimLeft(output, " \t\r\n") if strings.HasPrefix(trimmed, "{") || strings.HasPrefix(trimmed, "[") { - // JSON-ish output: prefer the last newline before the cap. + // Pretty-printed: prefer the last newline before the cap. if cut := strings.LastIndex(output[:maxChars], "\n"); cut >= 0 { - return output[:cut] + "\n... (truncated)" + return appendElisionMarker(output[:cut], output[cut:]) } - // Single-line JSON: splice at the previous element separator so the - // visible prefix remains well-formed up to the marker. - if cut := strings.LastIndex(output[:maxChars], ","); cut >= 0 { - return output[:cut+1] + "\n... (truncated)" + // Single-line JSON: splice at the last TOP-LEVEL element separator so + // each kept record stays complete (a naive comma search can land + // inside an object, orphaning half a record on both sides). + if cut := lastTopLevelComma(output, maxChars); cut > 0 { + return appendElisionMarker(output[:cut+1], output[cut+1:]) } // No safe splice: fall back to the byte cap. - return output[:maxChars] + "\n... (truncated)" + return appendElisionMarker(output[:maxChars], output[maxChars:]) } // Plain text: cut at the last line boundary to keep whole lines. if cut := strings.LastIndex(output[:maxChars], "\n"); cut > 0 { - return output[:cut] + "\n... (truncated)" + return appendElisionMarker(output[:cut], output[cut:]) + } + return appendElisionMarker(output[:maxChars], output[maxChars:]) +} + +// lastTopLevelComma returns the index of the last comma at bracket depth 1 +// within s[:limit] of an outer array/object, or -1. +func lastTopLevelComma(s string, limit int) int { + depth := 0 + inStr := false + esc := false + last := -1 + n := limit + if n > len(s) { + n = len(s) + } + for i := 0; i < n; i++ { + c := s[i] + if inStr { + switch { + case esc: + esc = false + case c == '\\': + esc = true + case c == '"': + inStr = false + } + continue + } + switch c { + case '"': + inStr = true + case '{', '[': + depth++ + case '}', ']': + depth-- + case ',': + if depth == 1 { + last = i + } + } } - return output[:maxChars] + "\n... (truncated)" + return last } // PostProcess applies the domain mutation/validation hooks that follow a raw diff --git a/internal/token/tok.go b/internal/token/tok.go index 08bddf5f..8c7cd383 100644 --- a/internal/token/tok.go +++ b/internal/token/tok.go @@ -4,6 +4,8 @@ package token import ( + "encoding/json" + tok "github.com/GrayCodeAI/tok" tokgraph "github.com/GrayCodeAI/tok/runtimegraph" ) @@ -31,6 +33,15 @@ func Compress(text string, budget int) (string, Stats) { func NewUsageTracker() *UsageTracker { return tok.NewUsageTracker() } +// JSONInvariants renders verified-fact summaries for elided JSON records +// (constants, enumerations, ranges, coverage). "" when nothing clears the +// withhold rules. +func JSONInvariants(dropped []json.RawMessage) string { return tok.JSONInvariants(dropped) } + +// LogInvariants renders the level distribution of elided log lines. +// "" when the lines do not parse as logs. +func LogInvariants(lines []string) string { return tok.LogInvariants(lines) } + func ChunkCode(source string, opts ChunkOptions) []CodeChunk { return tok.ChunkCode(source, opts) } From 8cdce9104cea2f9dd90771146bc9c53315e44e85 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sun, 23 Aug 2026 08:32:27 +0530 Subject: [PATCH 2/3] fix(deps): pin tok module to merged #82 commit for submodule parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release-parity check resolves each submodule's go.mod version and requires it to equal the index gitlink. Bumping the gitlink without updating the module requirement left CI resolving old tok (643b6675), which lacks JSONInvariants/LogInvariants — breaking module hygiene builds. Pin the require directive to a1d1863f (#82). --- external/tok | 2 +- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/external/tok b/external/tok index a1d1863f..a7c4b99d 160000 --- a/external/tok +++ b/external/tok @@ -1 +1 @@ -Subproject commit a1d1863f360d42a9eca34572d512e26c90c8bc22 +Subproject commit a7c4b99d37b8241d43e838f9d3c648a70fdc22f1 diff --git a/go.mod b/go.mod index 1d21abbf..2d070e80 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/GrayCodeAI/hawk-core-contracts v0.1.13-0.20260816034142-16ebcfd5ad6e github.com/GrayCodeAI/inspect v0.0.0-20260816041238-8556ee05ff07 github.com/GrayCodeAI/sight v0.0.0-20260816041235-39553454cd60 - github.com/GrayCodeAI/tok v0.1.5-0.20260816034249-643b6675ebc7 + github.com/GrayCodeAI/tok v0.1.5-0.20260823020239-a1d1863f360d github.com/GrayCodeAI/yaad v0.2.1-0.20260816034238-42bdda93995b github.com/alecthomas/chroma/v2 v2.26.1 github.com/bwmarrin/discordgo v0.28.1 diff --git a/go.sum b/go.sum index a52475b5..e7becd14 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/GrayCodeAI/sight v0.0.0-20260816041235-39553454cd60 h1:mXkSBokYHL83fT github.com/GrayCodeAI/sight v0.0.0-20260816041235-39553454cd60/go.mod h1:0D2fhnfizzjywVOx/QPIdMduOtv3nZt1xrC3SBYmaF8= github.com/GrayCodeAI/tok v0.1.5-0.20260816034249-643b6675ebc7 h1:agvfUOO5eVzCyI/mCYFr5Di8bB4OQSgEhrKgMyViTSA= github.com/GrayCodeAI/tok v0.1.5-0.20260816034249-643b6675ebc7/go.mod h1:zHM1Ei/uHq2793uVY5mEtli4o9znAwtfdzPgSCrTjyQ= +github.com/GrayCodeAI/tok v0.1.5-0.20260823020239-a1d1863f360d h1:i3V00Vjt+go8KL6bZDWNYHerlCgZZKNs5pQJumfKVHU= +github.com/GrayCodeAI/tok v0.1.5-0.20260823020239-a1d1863f360d/go.mod h1:zHM1Ei/uHq2793uVY5mEtli4o9znAwtfdzPgSCrTjyQ= github.com/GrayCodeAI/trace v0.1.4-0.20260816034253-59b437bbe8dc h1:b83/X8ETGFfu8Dn976v9ZLZJy6O1pVpInnqKc7i/TjU= github.com/GrayCodeAI/trace v0.1.4-0.20260816034253-59b437bbe8dc/go.mod h1:3IYIRSxM+ggLJmbzFM8undLSI0VaB7hFVkXzxRsyZaA= github.com/GrayCodeAI/yaad v0.2.1-0.20260816034238-42bdda93995b h1:ERRJu8E87qSA02j/9UM/3HqUNKSCYIQEnDWlQXHmix4= From cb65251162f6559b24d00bb1e65d80dcc08205af Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sun, 23 Aug 2026 08:51:06 +0530 Subject: [PATCH 3/3] fix(deps): point tok gitlink at #82 exactly; go work sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit carried the submodule working tree at a7c4b99d (toolschema #83) while pinning the module to a1d1863f (#82) — a parity mismatch. Pin the gitlink to a1d1863f on this branch; #83's pointer lands with the stacked wiring PR. Also commit go.work.sum drift from go work sync. --- external/tok | 2 +- go.sum | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/external/tok b/external/tok index a7c4b99d..a1d1863f 160000 --- a/external/tok +++ b/external/tok @@ -1 +1 @@ -Subproject commit a7c4b99d37b8241d43e838f9d3c648a70fdc22f1 +Subproject commit a1d1863f360d42a9eca34572d512e26c90c8bc22 diff --git a/go.sum b/go.sum index e7becd14..459c351b 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,6 @@ github.com/GrayCodeAI/inspect v0.0.0-20260816041238-8556ee05ff07 h1:XQRSF6Migl5X github.com/GrayCodeAI/inspect v0.0.0-20260816041238-8556ee05ff07/go.mod h1:ipnOyNHbY1I6H5BlZY4RBDmFBpRnuQbshwawTqzHS/8= github.com/GrayCodeAI/sight v0.0.0-20260816041235-39553454cd60 h1:mXkSBokYHL83fTM9i77n8ID2gSl5fUr2bISunzCw+CI= github.com/GrayCodeAI/sight v0.0.0-20260816041235-39553454cd60/go.mod h1:0D2fhnfizzjywVOx/QPIdMduOtv3nZt1xrC3SBYmaF8= -github.com/GrayCodeAI/tok v0.1.5-0.20260816034249-643b6675ebc7 h1:agvfUOO5eVzCyI/mCYFr5Di8bB4OQSgEhrKgMyViTSA= -github.com/GrayCodeAI/tok v0.1.5-0.20260816034249-643b6675ebc7/go.mod h1:zHM1Ei/uHq2793uVY5mEtli4o9znAwtfdzPgSCrTjyQ= github.com/GrayCodeAI/tok v0.1.5-0.20260823020239-a1d1863f360d h1:i3V00Vjt+go8KL6bZDWNYHerlCgZZKNs5pQJumfKVHU= github.com/GrayCodeAI/tok v0.1.5-0.20260823020239-a1d1863f360d/go.mod h1:zHM1Ei/uHq2793uVY5mEtli4o9znAwtfdzPgSCrTjyQ= github.com/GrayCodeAI/trace v0.1.4-0.20260816034253-59b437bbe8dc h1:b83/X8ETGFfu8Dn976v9ZLZJy6O1pVpInnqKc7i/TjU=