feat(elision): invariant-bearing truncation markers; bump tok to #82

internal/engine/elision.go

@@ -0,0 +1,77 @@
+package engine
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/GrayCodeAI/hawk/internal/engine/token"
+)
+
+// elisionNotice computes a verified-facts suffix for a truncation marker from
+// the content being dropped. Facts state only what holds across every elided
+// unit (see tok invariants): constants, exact enumerations, numeric ranges,
+// distinct-count coverage. Anything uncertain is omitted — a bare count is
+// better than a wrong fact, and a wrong fact reads as complete.
+//
+// JSON arrays of records get field-level facts; log-shaped text gets level
+// distribution; anything else falls back to the line count.
+func elisionNotice(dropped string) string {
+	trimmed := strings.TrimSpace(dropped)
+	if trimmed == "" {
+		return ""
+	}
+	if strings.HasPrefix(trimmed, "[") {
+		return jsonRecordsFacts(trimmed)
+	}
+	// A structural cut inside a JSON array leaves a fragment of whole records
+	// separated by commas, optionally ending with the array's closing bracket
+	// ("{…},{…}]" or "{…},{…},"). Strip both artifacts, normalize to an
+	// array, and retry once; anything unparseable still falls back.
+	if strings.HasPrefix(trimmed, "{") {
+		frag := strings.TrimRight(trimmed, " \t\r\n")
+		frag = strings.TrimSuffix(frag, "]")
+		frag = strings.TrimRight(frag, ", \t\r\n")
+		if facts := jsonRecordsFacts("[" + frag + "]"); facts != "" {
+			return facts
+		}
+	}
+	lines := splitNonEmptyLines(trimmed)
+	if len(lines) >= 3 {
+		if facts := token.LogInvariants(lines); facts != "" {
+			return fmt.Sprintf("%d lines elided: %s", len(lines), facts)
+		}
+		return fmt.Sprintf("%d lines elided", len(lines))
+	}
+	return ""
+}
+
+func jsonRecordsFacts(arrayText string) string {
+	var records []json.RawMessage
+	if json.Unmarshal([]byte(arrayText), &records) != nil || len(records) == 0 {
+		return ""
+	}
+	if facts := token.JSONInvariants(records); facts != "" {
+		return fmt.Sprintf("%d records elided: %s", len(records), facts)
+	}
+	return fmt.Sprintf("%d records elided", len(records))
+}
+
+func splitNonEmptyLines(s string) []string {
+	raw := strings.Split(s, "\n")
+	out := make([]string, 0, len(raw))
+	for _, l := range raw {
+		if t := strings.TrimSpace(l); t != "" {
+			out = append(out, t)
+		}
+	}
+	return out
+}
+
+// appendElisionMarker attaches an invariant-bearing marker to kept output.
+func appendElisionMarker(kept, dropped string) string {
+	if n := elisionNotice(dropped); n != "" {
+		return kept + "\n... [" + n + "]"
+	}
+	return kept + "\n... (truncated)"
+}

internal/engine/elision_test.go

@@ -0,0 +1,74 @@
+package engine
+
+import (
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestElisionNoticeJSONRecords(t *testing.T) {
+	var items []string
+	for i := 0; i < 10; i++ {
+		items = append(items, `{"order_id":"ord-`+strconv.Itoa(i)+`","status":"fulfilled"}`)
+	}
+	notice := elisionNotice("[" + strings.Join(items, ",") + "]")
+	if !strings.Contains(notice, "records elided") {
+		t.Fatalf("notice = %q", notice)
+	}
+	if !strings.Contains(notice, "status=fulfilled×") && !strings.Contains(notice, "distinct") {
+		t.Fatalf("no verified facts: %q", notice)
+	}
+}
+
+func TestElisionNoticeLogLines(t *testing.T) {
+	var lines []string
+	for i := 0; i < 6; i++ {
+		lines = append(lines, "2026-08-22T10:00:0"+strconv.Itoa(i)+"Z INFO tick")
+	}
+	notice := elisionNotice(strings.Join(lines, "\n"))
+	if !strings.Contains(notice, "lines elided") || !strings.Contains(notice, "info×6") {
+		t.Fatalf("notice = %q", notice)
+	}
+}
+
+func TestElisionNoticeProseFallback(t *testing.T) {
+	notice := elisionNotice("just two\nshort lines")
+	if notice != "" {
+		t.Fatalf("tiny prose should yield no notice, got %q", notice)
+	}
+}
+
+func TestAppendElisionMarker(t *testing.T) {
+	var items []string
+	for i := 0; i < 12; i++ {
+		items = append(items, `{"id":"`+strconv.Itoa(i)+`","state":"ok"}`)
+	}
+	dropped := "[" + strings.Join(items[3:], ",") + "]"
+	out := appendElisionMarker(`[{"id":"0","state":"ok"},{"id":"1","state":"ok"},{"id":"2","state":"ok"}]`, dropped)
+	if !strings.Contains(out, "_tok") == false && !strings.Contains(out, "records elided") {
+		t.Fatalf("marker missing facts: %q", out)
+	}
+	// kept content preserved verbatim before marker
+	if !strings.HasPrefix(out, `[{"id":"0","state":"ok"}`) {
+		t.Fatalf("kept content altered: %q", out)
+	}
+}
+
+func TestTruncateToolOutputCarriesInvariants(t *testing.T) {
+	var items []string
+	for i := 100; i < 160; i++ {
+		items = append(items, `{"sku":"sku-`+strconv.Itoa(i)+`","status":"shipped"}`)
+	}
+	output := "[" + strings.Join(items, ",") + "]"
+	// The structural cutter keeps whole records, so the dropped tail parses
+	// as a clean record set and invariants can be computed. A raw byte cut
+	// landing mid-record correctly falls back to the bare marker.
+	got := truncateOutputStructurally(output, 500)
+	if !strings.Contains(got, "records elided") {
+		t.Fatalf("got %q", got)
+	}
+	// The kept prefix must remain valid JSON prefix content (starts with '[').
+	if !strings.HasPrefix(strings.TrimSpace(got), "[") {
+		t.Fatal("lost array opening")
+	}
+}

internal/engine/token/tok_facade.go

@@ -1,6 +1,10 @@
 package token
 
-import hawktoken "github.com/GrayCodeAI/hawk/internal/token"
+import (
+	"encoding/json"
+
+	hawktoken "github.com/GrayCodeAI/hawk/internal/token"
+)
 
 // Stats is the compression result consumed by Hawk's runtime observations.
 // The alias preserves the external tok schema while keeping Tok imports inside
@@ -41,3 +45,9 @@ func BuildRuntimeGraph(input RuntimeGraphInput) (*RuntimeGraphExport, error) {
 func Compress(text string, budget int) (string, Stats) {
 	return hawktoken.Compress(text, budget)
 }
+
+// JSONInvariants renders verified-fact summaries for elided JSON records.
+func JSONInvariants(dropped []json.RawMessage) string { return hawktoken.JSONInvariants(dropped) }
+
+// LogInvariants renders the level distribution of elided log lines.
+func LogInvariants(lines []string) string { return hawktoken.LogInvariants(lines) }

internal/engine/tool_output_spill.go

@@ -51,7 +51,12 @@ func maybeSpillToolOutput(output, toolName, toolID string) string {
 	}
 	preview := output
 	if len(preview) > toolOutputSpillPreview {
-		preview = preview[:toolOutputSpillPreview] + "\n... (see file for full output)"
+		preview = preview[:toolOutputSpillPreview]
+		if n := elisionNotice(output[toolOutputSpillPreview:]); n != "" {
+			preview += "\n... [" + n + "; full output in file]"
+		} else {
+			preview += "\n... (see file for full output)"
+		}
 	}
 	return fmt.Sprintf(
 		"Output saved to %s (%d bytes).\n\nPreview:\n%s\n\nUse Read (offset/limit) or Grep on this path to inspect the rest.",
@@ -63,5 +68,5 @@ func truncateToolOutput(output string, max int) string {
 	if len(output) <= max {
 		return output
 	}
-	return output[:max] + "\n... (truncated)"
+	return appendElisionMarker(output[:max], output[max:])
 }

internal/engine/tool_service.go

@@ -591,27 +591,68 @@ func (s *ToolService) NormalizeOutput(output, canonicalTool, toolID string, cont
 
 // truncateOutputStructurally trims oversized tool output at a structural
 // boundary instead of a raw byte cut, so JSON-ish results keep whole lines
-// (or a valid splice point) rather than being chopped mid-object (Phase 3).
+// (or whole array elements) rather than being chopped mid-object (Phase 3).
 func truncateOutputStructurally(output string, maxChars int) string {
 	trimmed := strings.TrimLeft(output, " \t\r\n")
 	if strings.HasPrefix(trimmed, "{") || strings.HasPrefix(trimmed, "[") {
-		// JSON-ish output: prefer the last newline before the cap.
+		// Pretty-printed: prefer the last newline before the cap.
 		if cut := strings.LastIndex(output[:maxChars], "\n"); cut >= 0 {
-			return output[:cut] + "\n... (truncated)"
+			return appendElisionMarker(output[:cut], output[cut:])
 		}
-		// Single-line JSON: splice at the previous element separator so the
-		// visible prefix remains well-formed up to the marker.
-		if cut := strings.LastIndex(output[:maxChars], ","); cut >= 0 {
-			return output[:cut+1] + "\n... (truncated)"
+		// Single-line JSON: splice at the last TOP-LEVEL element separator so
+		// each kept record stays complete (a naive comma search can land
+		// inside an object, orphaning half a record on both sides).
+		if cut := lastTopLevelComma(output, maxChars); cut > 0 {
+			return appendElisionMarker(output[:cut+1], output[cut+1:])
 		}
 		// No safe splice: fall back to the byte cap.
-		return output[:maxChars] + "\n... (truncated)"
+		return appendElisionMarker(output[:maxChars], output[maxChars:])
 	}
 	// Plain text: cut at the last line boundary to keep whole lines.
 	if cut := strings.LastIndex(output[:maxChars], "\n"); cut > 0 {
-		return output[:cut] + "\n... (truncated)"
+		return appendElisionMarker(output[:cut], output[cut:])
+	}
+	return appendElisionMarker(output[:maxChars], output[maxChars:])
+}
+
+// lastTopLevelComma returns the index of the last comma at bracket depth 1
+// within s[:limit] of an outer array/object, or -1.
+func lastTopLevelComma(s string, limit int) int {
+	depth := 0
+	inStr := false
+	esc := false
+	last := -1
+	n := limit
+	if n > len(s) {
+		n = len(s)
+	}
+	for i := 0; i < n; i++ {
+		c := s[i]
+		if inStr {
+			switch {
+			case esc:
+				esc = false
+			case c == '\\':
+				esc = true
+			case c == '"':
+				inStr = false
+			}
+			continue
+		}
+		switch c {
+		case '"':
+			inStr = true
+		case '{', '[':
+			depth++
+		case '}', ']':
+			depth--
+		case ',':
+			if depth == 1 {
+				last = i
+			}
+		}
 	}
-	return output[:maxChars] + "\n... (truncated)"
+	return last
 }
 
 // PostProcess applies the domain mutation/validation hooks that follow a raw

internal/token/tok.go

@@ -4,6 +4,8 @@
 package token
 
 import (
+	"encoding/json"
+
 	tok "github.com/GrayCodeAI/tok"
 	tokgraph "github.com/GrayCodeAI/tok/runtimegraph"
 )
@@ -31,6 +33,15 @@ func Compress(text string, budget int) (string, Stats) {
 
 func NewUsageTracker() *UsageTracker { return tok.NewUsageTracker() }
 
+// JSONInvariants renders verified-fact summaries for elided JSON records
+// (constants, enumerations, ranges, coverage). "" when nothing clears the
+// withhold rules.
+func JSONInvariants(dropped []json.RawMessage) string { return tok.JSONInvariants(dropped) }
+
+// LogInvariants renders the level distribution of elided log lines.
+// "" when the lines do not parse as logs.
+func LogInvariants(lines []string) string { return tok.LogInvariants(lines) }
+
 func ChunkCode(source string, opts ChunkOptions) []CodeChunk {
 	return tok.ChunkCode(source, opts)
 }