Split the sections that hold the evidence, on a Judge (HAL-1374)

cmd/navbench/main.go

@@ -71,7 +71,7 @@ func main() {
 	trees := flag.String("trees", "", "directory of tocdump trees")
 	pdfs := flag.String("pdfs", "", "directory of filings")
 	out := flag.String("out", "", "JSONL of per-question outcomes")
-	maxLeaves := flag.Int("leaves", 3, "sections read per question")
+	maxLeaves := flag.Int("leaves", 0, "cap on sections read per question; 0 lets the page budget decide")
 	maxPages := flag.Int("pages", 40, "pages judged per question")
 	limitQ := flag.Int("limit", 0, "stop after this many questions (0 = all)")
 	parallel := flag.Int("parallel", 1, "questions in flight at once; the provider's adaptive limiter governs requests")

cmd/tocdump/main.go

@@ -61,6 +61,7 @@ func main() {
 	// timeout there silently drops the whole tree. Measured 2026-09-18.
 	callTimeout := flag.Duration("timeout", 300*time.Second, "per LLM call timeout")
 	parallel := flag.Int("parallel", 1, "documents in flight at once; the provider's adaptive limiter governs requests")
+	split := flag.Int("split", 0, "split leaves spanning more than this many pages into sub-leaves (0 = default 20, negative = off)")
 	flag.Parse()
 	if *parallel < 1 {
 		*parallel = 1
@@ -119,7 +120,7 @@ func main() {
 			if *judgeOnly {
 				llm = refusingClient{}
 			}
-			b := &ingest.TOCBuilder{LLM: llm, Judge: judge, LLMCallTimeout: *callTimeout, MinimalContext: *minimal}
+			b := &ingest.TOCBuilder{LLM: llm, Judge: judge, LLMCallTimeout: *callTimeout, MinimalContext: *minimal, SplitLeavesOver: *split}
 			ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
 			start := time.Now()
 			nodes, usage, err := b.Build(ctx, pages)

docs/evaluations/2026-09-19-leaf-granularity.md

@@ -0,0 +1,80 @@
+# Leaf granularity — splitting the sections that hold the evidence
+
+**Date:** 2026-09-19
+**Harness:** [`cmd/tocdump -split`](../../cmd/tocdump/main.go), [`cmd/tocdump/coverage.py`](../../cmd/tocdump/coverage.py), [`cmd/tocdump/titles.py`](../../cmd/tocdump/titles.py), [`cmd/navbench`](../../cmd/navbench/main.go)
+**Corpus:** FinanceBench, 21 10-K filings, 47 gold evidence pages, 40 questions
+**Issue:** HAL-1374
+**Question:** a 10-K's tree is fine where nothing is (Items 1B–4, a paragraph each) and coarse where everything is (Item 8 and its notes, 70 pages under one title). Does splitting the big leaves at their own headings, on the Judge, make retrieval find more and cite tighter?
+
+## Result: right section 39 → 40 of 40; the leaf holding a gold page shrinks from 37 pages to 6 at the median; pages found stays at 36 of 40. Splitting defaults on at 20 pages.
+
+| tree | leaves / filing | span of the leaf holding a gold page, median | gold pages inside a leaf | right section | every gold page found | requests / question | $ / question |
+|---|---|---|---|---|---|---|---|
+| unsplit (contents-page grain) | 23 | **37 p** | 47 / 47 | 39 / 40 | 36 / 40 | 4.2 | 0.0035 |
+| split over 8 | 69 | 1 p | 47 / 47 | **40 / 40** | 36 / 40 | 4.3 | 0.0041 |
+| split over 12 | 67 | 1 p | 47 / 47 | **40 / 40** | 35 / 40 | 4.4 | 0.0041 |
+| split over 20 | 68 | 1 p | 47 / 47 | **40 / 40** | 36 / 40 | 4.3 | 0.0040 |
+| split over 12, tightened budget | 38 | 7 p | **39 / 47** | 33 / 40 | 30 / 40 | 4.2 | 0.0037 |
+| **split over 20, final** | **54** | **6 p** | **47 / 47** | **40 / 40** | **36 / 40** | 4.3 | 0.0039 |
+
+All navigation rows use the navigator after the fix described below.
+The TOC stage pays for the split: 271 Judge requests over the corpus
+against 63, about $0.006 per filing against $0.0007, and 489 s wall at
+parallel 8 against 122 s. Leaf titles against the unsplit tree: recall
+0.971 (17 of 589 lost to sub-leaf re-titling), precision 0.542 — every
+sub-leaf is an "extra" by construction.
+
+## What the sweep taught, in order
+
+1. **The first pass made navigation worse, and the tree was not at
+   fault.** At T=12 the right-section rate fell 37 → 33 and pages read
+   fell 40 → 16. The navigator took a fixed five sections; on a tree of
+   80 one-page notes that was five pages. Sections are now taken in
+   rank order until the page budget is gathered. Same trees: right
+   section 33 → 40, evidence 31 → 35. (The bench's own `-leaves 3`
+   default masked the fix for one run.)
+2. **Every threshold produced about the same leaf count: 69, 67 and 68
+   per filing (medians) at 8, 12 and 20.** The per-page cap, not the
+   content, was deciding, and it kept the earliest headings rather than
+   the best. Sub-leaves are now kept by
+   the Judge's probability.
+3. **Tightening the budget to one sub-leaf per half-threshold lost
+   coverage: 47 → 39 of 47.** Not because headings were dropped — because
+   sub-leaves began at the first confirmed heading, so the pages before
+   it (Item 8's index and the auditor's report) belonged to no leaf. A
+   gold page there was inside nothing. The section's opening is now its
+   first sub-leaf, titled as the parent. The loose budget was restored.
+4. **8, 12 and 20 navigate alike.** 20 makes the fewest sub-leaves and
+   costs the least, so it is the default whenever a Judge is set.
+
+## Where the sub-leaves come from
+
+Two sources, in order of trust, both "select, don't generate":
+
+- **A nested contents page** inside the leaf — Item 8 opens with an
+  index of statements and notes with page numbers. Parsed, confirmed and
+  resolved exactly as the document's own contents page is.
+- **Heading-shaped lines** — short, line-opening, Title Case or capitals,
+  not repeated on three or more pages (a running header is not a
+  heading). Each is judged with the lines that follow it: *is this a
+  heading a reader could turn to, rather than page furniture?* Boeing's
+  Item 7 became fifteen sub-sections from "Consolidated Results of
+  Operations" to "Contingent Obligations"; its Item 8 became the six
+  statements and twenty-one notes.
+
+## What is still missed
+
+Four questions, the same four as before the split: Boeing's legal
+proceedings (the "see Note 21" hop now has a Note 21 leaf to reach, and
+still misses the page inside it), and three page-level misses inside
+the right section. Section choice is solved on this corpus; the
+full-page question, or what it is shown, is next.
+
+## Reproduce
+
+```bash
+go run ./cmd/tocdump -docs ~/.cache/vlbench/financebench -out /tmp/t -judge-only -minimal -parallel 8          # split on by default
+go run ./cmd/tocdump -docs ~/.cache/vlbench/financebench -out /tmp/t0 -judge-only -minimal -parallel 8 -split -1  # off
+python cmd/tocdump/coverage.py /tmp/t0 /tmp/t
+go run ./cmd/navbench -questions ~/.cache/vlbench/financebench-questions.jsonl -trees /tmp/t -pdfs ~/.cache/vlbench/financebench -parallel 4
+```

pkg/ingest/toc_builder.go

@@ -105,6 +105,12 @@ type TOCBuilder struct {
 	// to earn its place.
 	MinimalContext bool
 
+	// SplitLeavesOver splits any leaf spanning more pages than this into
+	// sub-leaves at its internal headings, confirmed by the Judge
+	// (HAL-1374). Zero selects the default of 20 when a Judge is set;
+	// negative disables splitting.
+	SplitLeavesOver int
+
 	// DetectChars caps the characters of each page sent to detection when
 	// MinimalContext is on. Zero means detectCharsMinimal. A contents page
 	// declares itself in its first couple of thousand characters; the
@@ -272,6 +278,15 @@ func (b *TOCBuilder) Build(ctx context.Context, pages []PageText) ([]tree.TOCNod
 	// start pages drive the derivation.
 	deriveEndPages(nodes, lastPage(pages))
 
+	// Split the leaves that are too big to cite or to read, at their
+	// own internal headings (HAL-1374). Needs the spans, so it runs
+	// after end pages; adds its own children's end pages.
+	if over := b.splitLeavesOver(); over > 0 {
+		if n := b.splitLargeLeaves(ctx, nodes, pages, over, &usage); n > 0 {
+			log.Printf("toc: %d sub-leaves added inside leaves over %d pages", n, over)
+		}
+	}
+
 	// Stamp stable node IDs onto every node so callers / external
 	// consumers have an opaque handle independent of position.
 	stampNodeIDs(nodes, "")
@@ -288,6 +303,21 @@ const resolverAttempts = 2
 // exhaustion it leaves the tree exactly as extraction produced it and
 // records the fact; it never routes a Judge-path document through the
 // generative verifier.
+// defaultSplitLeavesOver: measured on FinanceBench, splitting at 8, 12
+// and 20 pages navigated alike (right section 40/40, evidence 36/40);
+// 20 makes the fewest sub-leaves and costs the least.
+const defaultSplitLeavesOver = 20
+
+func (b *TOCBuilder) splitLeavesOver() int {
+	switch {
+	case b.SplitLeavesOver < 0, b.Judge == nil:
+		return 0
+	case b.SplitLeavesOver == 0:
+		return defaultSplitLeavesOver
+	}
+	return b.SplitLeavesOver
+}
+
 func (b *TOCBuilder) resolvePagesOrKeep(ctx context.Context, nodes []tree.TOCNode, pages []PageText, exclude []int, usage *Usage) {
 	var lastErr error
 	for attempt := 1; attempt <= resolverAttempts; attempt++ {