diff --git a/external/tok b/external/tok index 643b6675..a1d1863f 160000 --- a/external/tok +++ b/external/tok @@ -1 +1 @@ -Subproject commit 643b6675ebc75b32e448fbebcd3caba92b6d7583 +Subproject commit a1d1863f360d42a9eca34572d512e26c90c8bc22 diff --git a/go.mod b/go.mod index 1d21abbf..2d070e80 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/GrayCodeAI/hawk-core-contracts v0.1.13-0.20260816034142-16ebcfd5ad6e github.com/GrayCodeAI/inspect v0.0.0-20260816041238-8556ee05ff07 github.com/GrayCodeAI/sight v0.0.0-20260816041235-39553454cd60 - github.com/GrayCodeAI/tok v0.1.5-0.20260816034249-643b6675ebc7 + github.com/GrayCodeAI/tok v0.1.5-0.20260823020239-a1d1863f360d github.com/GrayCodeAI/yaad v0.2.1-0.20260816034238-42bdda93995b github.com/alecthomas/chroma/v2 v2.26.1 github.com/bwmarrin/discordgo v0.28.1 diff --git a/go.sum b/go.sum index a52475b5..459c351b 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/GrayCodeAI/inspect v0.0.0-20260816041238-8556ee05ff07 h1:XQRSF6Migl5X github.com/GrayCodeAI/inspect v0.0.0-20260816041238-8556ee05ff07/go.mod h1:ipnOyNHbY1I6H5BlZY4RBDmFBpRnuQbshwawTqzHS/8= github.com/GrayCodeAI/sight v0.0.0-20260816041235-39553454cd60 h1:mXkSBokYHL83fTM9i77n8ID2gSl5fUr2bISunzCw+CI= github.com/GrayCodeAI/sight v0.0.0-20260816041235-39553454cd60/go.mod h1:0D2fhnfizzjywVOx/QPIdMduOtv3nZt1xrC3SBYmaF8= -github.com/GrayCodeAI/tok v0.1.5-0.20260816034249-643b6675ebc7 h1:agvfUOO5eVzCyI/mCYFr5Di8bB4OQSgEhrKgMyViTSA= -github.com/GrayCodeAI/tok v0.1.5-0.20260816034249-643b6675ebc7/go.mod h1:zHM1Ei/uHq2793uVY5mEtli4o9znAwtfdzPgSCrTjyQ= +github.com/GrayCodeAI/tok v0.1.5-0.20260823020239-a1d1863f360d h1:i3V00Vjt+go8KL6bZDWNYHerlCgZZKNs5pQJumfKVHU= +github.com/GrayCodeAI/tok v0.1.5-0.20260823020239-a1d1863f360d/go.mod h1:zHM1Ei/uHq2793uVY5mEtli4o9znAwtfdzPgSCrTjyQ= github.com/GrayCodeAI/trace v0.1.4-0.20260816034253-59b437bbe8dc h1:b83/X8ETGFfu8Dn976v9ZLZJy6O1pVpInnqKc7i/TjU= github.com/GrayCodeAI/trace v0.1.4-0.20260816034253-59b437bbe8dc/go.mod h1:3IYIRSxM+ggLJmbzFM8undLSI0VaB7hFVkXzxRsyZaA= github.com/GrayCodeAI/yaad v0.2.1-0.20260816034238-42bdda93995b h1:ERRJu8E87qSA02j/9UM/3HqUNKSCYIQEnDWlQXHmix4= diff --git a/internal/engine/elision.go b/internal/engine/elision.go new file mode 100644 index 00000000..11470af4 --- /dev/null +++ b/internal/engine/elision.go @@ -0,0 +1,77 @@ +package engine + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/GrayCodeAI/hawk/internal/engine/token" +) + +// elisionNotice computes a verified-facts suffix for a truncation marker from +// the content being dropped. Facts state only what holds across every elided +// unit (see tok invariants): constants, exact enumerations, numeric ranges, +// distinct-count coverage. Anything uncertain is omitted — a bare count is +// better than a wrong fact, and a wrong fact reads as complete. +// +// JSON arrays of records get field-level facts; log-shaped text gets level +// distribution; anything else falls back to the line count. +func elisionNotice(dropped string) string { + trimmed := strings.TrimSpace(dropped) + if trimmed == "" { + return "" + } + if strings.HasPrefix(trimmed, "[") { + return jsonRecordsFacts(trimmed) + } + // A structural cut inside a JSON array leaves a fragment of whole records + // separated by commas, optionally ending with the array's closing bracket + // ("{…},{…}]" or "{…},{…},"). Strip both artifacts, normalize to an + // array, and retry once; anything unparseable still falls back. + if strings.HasPrefix(trimmed, "{") { + frag := strings.TrimRight(trimmed, " \t\r\n") + frag = strings.TrimSuffix(frag, "]") + frag = strings.TrimRight(frag, ", \t\r\n") + if facts := jsonRecordsFacts("[" + frag + "]"); facts != "" { + return facts + } + } + lines := splitNonEmptyLines(trimmed) + if len(lines) >= 3 { + if facts := token.LogInvariants(lines); facts != "" { + return fmt.Sprintf("%d lines elided: %s", len(lines), facts) + } + return fmt.Sprintf("%d lines elided", len(lines)) + } + return "" +} + +func jsonRecordsFacts(arrayText string) string { + var records []json.RawMessage + if json.Unmarshal([]byte(arrayText), &records) != nil || len(records) == 0 { + return "" + } + if facts := token.JSONInvariants(records); facts != "" { + return fmt.Sprintf("%d records elided: %s", len(records), facts) + } + return fmt.Sprintf("%d records elided", len(records)) +} + +func splitNonEmptyLines(s string) []string { + raw := strings.Split(s, "\n") + out := make([]string, 0, len(raw)) + for _, l := range raw { + if t := strings.TrimSpace(l); t != "" { + out = append(out, t) + } + } + return out +} + +// appendElisionMarker attaches an invariant-bearing marker to kept output. +func appendElisionMarker(kept, dropped string) string { + if n := elisionNotice(dropped); n != "" { + return kept + "\n... [" + n + "]" + } + return kept + "\n... (truncated)" +} diff --git a/internal/engine/elision_test.go b/internal/engine/elision_test.go new file mode 100644 index 00000000..49db83fe --- /dev/null +++ b/internal/engine/elision_test.go @@ -0,0 +1,74 @@ +package engine + +import ( + "strconv" + "strings" + "testing" +) + +func TestElisionNoticeJSONRecords(t *testing.T) { + var items []string + for i := 0; i < 10; i++ { + items = append(items, `{"order_id":"ord-`+strconv.Itoa(i)+`","status":"fulfilled"}`) + } + notice := elisionNotice("[" + strings.Join(items, ",") + "]") + if !strings.Contains(notice, "records elided") { + t.Fatalf("notice = %q", notice) + } + if !strings.Contains(notice, "status=fulfilled×") && !strings.Contains(notice, "distinct") { + t.Fatalf("no verified facts: %q", notice) + } +} + +func TestElisionNoticeLogLines(t *testing.T) { + var lines []string + for i := 0; i < 6; i++ { + lines = append(lines, "2026-08-22T10:00:0"+strconv.Itoa(i)+"Z INFO tick") + } + notice := elisionNotice(strings.Join(lines, "\n")) + if !strings.Contains(notice, "lines elided") || !strings.Contains(notice, "info×6") { + t.Fatalf("notice = %q", notice) + } +} + +func TestElisionNoticeProseFallback(t *testing.T) { + notice := elisionNotice("just two\nshort lines") + if notice != "" { + t.Fatalf("tiny prose should yield no notice, got %q", notice) + } +} + +func TestAppendElisionMarker(t *testing.T) { + var items []string + for i := 0; i < 12; i++ { + items = append(items, `{"id":"`+strconv.Itoa(i)+`","state":"ok"}`) + } + dropped := "[" + strings.Join(items[3:], ",") + "]" + out := appendElisionMarker(`[{"id":"0","state":"ok"},{"id":"1","state":"ok"},{"id":"2","state":"ok"}]`, dropped) + if !strings.Contains(out, "_tok") == false && !strings.Contains(out, "records elided") { + t.Fatalf("marker missing facts: %q", out) + } + // kept content preserved verbatim before marker + if !strings.HasPrefix(out, `[{"id":"0","state":"ok"}`) { + t.Fatalf("kept content altered: %q", out) + } +} + +func TestTruncateToolOutputCarriesInvariants(t *testing.T) { + var items []string + for i := 100; i < 160; i++ { + items = append(items, `{"sku":"sku-`+strconv.Itoa(i)+`","status":"shipped"}`) + } + output := "[" + strings.Join(items, ",") + "]" + // The structural cutter keeps whole records, so the dropped tail parses + // as a clean record set and invariants can be computed. A raw byte cut + // landing mid-record correctly falls back to the bare marker. + got := truncateOutputStructurally(output, 500) + if !strings.Contains(got, "records elided") { + t.Fatalf("got %q", got) + } + // The kept prefix must remain valid JSON prefix content (starts with '['). + if !strings.HasPrefix(strings.TrimSpace(got), "[") { + t.Fatal("lost array opening") + } +} diff --git a/internal/engine/token/tok_facade.go b/internal/engine/token/tok_facade.go index f5c95bcb..91d72377 100644 --- a/internal/engine/token/tok_facade.go +++ b/internal/engine/token/tok_facade.go @@ -1,6 +1,10 @@ package token -import hawktoken "github.com/GrayCodeAI/hawk/internal/token" +import ( + "encoding/json" + + hawktoken "github.com/GrayCodeAI/hawk/internal/token" +) // Stats is the compression result consumed by Hawk's runtime observations. // The alias preserves the external tok schema while keeping Tok imports inside @@ -41,3 +45,9 @@ func BuildRuntimeGraph(input RuntimeGraphInput) (*RuntimeGraphExport, error) { func Compress(text string, budget int) (string, Stats) { return hawktoken.Compress(text, budget) } + +// JSONInvariants renders verified-fact summaries for elided JSON records. +func JSONInvariants(dropped []json.RawMessage) string { return hawktoken.JSONInvariants(dropped) } + +// LogInvariants renders the level distribution of elided log lines. +func LogInvariants(lines []string) string { return hawktoken.LogInvariants(lines) } diff --git a/internal/engine/tool_output_spill.go b/internal/engine/tool_output_spill.go index a38e4498..7d43e0c3 100644 --- a/internal/engine/tool_output_spill.go +++ b/internal/engine/tool_output_spill.go @@ -51,7 +51,12 @@ func maybeSpillToolOutput(output, toolName, toolID string) string { } preview := output if len(preview) > toolOutputSpillPreview { - preview = preview[:toolOutputSpillPreview] + "\n... (see file for full output)" + preview = preview[:toolOutputSpillPreview] + if n := elisionNotice(output[toolOutputSpillPreview:]); n != "" { + preview += "\n... [" + n + "; full output in file]" + } else { + preview += "\n... (see file for full output)" + } } return fmt.Sprintf( "Output saved to %s (%d bytes).\n\nPreview:\n%s\n\nUse Read (offset/limit) or Grep on this path to inspect the rest.", @@ -63,5 +68,5 @@ func truncateToolOutput(output string, max int) string { if len(output) <= max { return output } - return output[:max] + "\n... (truncated)" + return appendElisionMarker(output[:max], output[max:]) } diff --git a/internal/engine/tool_service.go b/internal/engine/tool_service.go index 71f63473..b32b9ba5 100644 --- a/internal/engine/tool_service.go +++ b/internal/engine/tool_service.go @@ -591,27 +591,68 @@ func (s *ToolService) NormalizeOutput(output, canonicalTool, toolID string, cont // truncateOutputStructurally trims oversized tool output at a structural // boundary instead of a raw byte cut, so JSON-ish results keep whole lines -// (or a valid splice point) rather than being chopped mid-object (Phase 3). +// (or whole array elements) rather than being chopped mid-object (Phase 3). func truncateOutputStructurally(output string, maxChars int) string { trimmed := strings.TrimLeft(output, " \t\r\n") if strings.HasPrefix(trimmed, "{") || strings.HasPrefix(trimmed, "[") { - // JSON-ish output: prefer the last newline before the cap. + // Pretty-printed: prefer the last newline before the cap. if cut := strings.LastIndex(output[:maxChars], "\n"); cut >= 0 { - return output[:cut] + "\n... (truncated)" + return appendElisionMarker(output[:cut], output[cut:]) } - // Single-line JSON: splice at the previous element separator so the - // visible prefix remains well-formed up to the marker. - if cut := strings.LastIndex(output[:maxChars], ","); cut >= 0 { - return output[:cut+1] + "\n... (truncated)" + // Single-line JSON: splice at the last TOP-LEVEL element separator so + // each kept record stays complete (a naive comma search can land + // inside an object, orphaning half a record on both sides). + if cut := lastTopLevelComma(output, maxChars); cut > 0 { + return appendElisionMarker(output[:cut+1], output[cut+1:]) } // No safe splice: fall back to the byte cap. - return output[:maxChars] + "\n... (truncated)" + return appendElisionMarker(output[:maxChars], output[maxChars:]) } // Plain text: cut at the last line boundary to keep whole lines. if cut := strings.LastIndex(output[:maxChars], "\n"); cut > 0 { - return output[:cut] + "\n... (truncated)" + return appendElisionMarker(output[:cut], output[cut:]) + } + return appendElisionMarker(output[:maxChars], output[maxChars:]) +} + +// lastTopLevelComma returns the index of the last comma at bracket depth 1 +// within s[:limit] of an outer array/object, or -1. +func lastTopLevelComma(s string, limit int) int { + depth := 0 + inStr := false + esc := false + last := -1 + n := limit + if n > len(s) { + n = len(s) + } + for i := 0; i < n; i++ { + c := s[i] + if inStr { + switch { + case esc: + esc = false + case c == '\\': + esc = true + case c == '"': + inStr = false + } + continue + } + switch c { + case '"': + inStr = true + case '{', '[': + depth++ + case '}', ']': + depth-- + case ',': + if depth == 1 { + last = i + } + } } - return output[:maxChars] + "\n... (truncated)" + return last } // PostProcess applies the domain mutation/validation hooks that follow a raw diff --git a/internal/token/tok.go b/internal/token/tok.go index 08bddf5f..8c7cd383 100644 --- a/internal/token/tok.go +++ b/internal/token/tok.go @@ -4,6 +4,8 @@ package token import ( + "encoding/json" + tok "github.com/GrayCodeAI/tok" tokgraph "github.com/GrayCodeAI/tok/runtimegraph" ) @@ -31,6 +33,15 @@ func Compress(text string, budget int) (string, Stats) { func NewUsageTracker() *UsageTracker { return tok.NewUsageTracker() } +// JSONInvariants renders verified-fact summaries for elided JSON records +// (constants, enumerations, ranges, coverage). "" when nothing clears the +// withhold rules. +func JSONInvariants(dropped []json.RawMessage) string { return tok.JSONInvariants(dropped) } + +// LogInvariants renders the level distribution of elided log lines. +// "" when the lines do not parse as logs. +func LogInvariants(lines []string) string { return tok.LogInvariants(lines) } + func ChunkCode(source string, opts ChunkOptions) []CodeChunk { return tok.ChunkCode(source, opts) }