diff --git a/cmd/navbench/main.go b/cmd/navbench/main.go index be38a86..e8f541c 100644 --- a/cmd/navbench/main.go +++ b/cmd/navbench/main.go @@ -71,7 +71,7 @@ func main() { trees := flag.String("trees", "", "directory of tocdump trees") pdfs := flag.String("pdfs", "", "directory of filings") out := flag.String("out", "", "JSONL of per-question outcomes") - maxLeaves := flag.Int("leaves", 3, "sections read per question") + maxLeaves := flag.Int("leaves", 0, "cap on sections read per question; 0 lets the page budget decide") maxPages := flag.Int("pages", 40, "pages judged per question") limitQ := flag.Int("limit", 0, "stop after this many questions (0 = all)") parallel := flag.Int("parallel", 1, "questions in flight at once; the provider's adaptive limiter governs requests") diff --git a/cmd/tocdump/main.go b/cmd/tocdump/main.go index fac564e..13f17b3 100644 --- a/cmd/tocdump/main.go +++ b/cmd/tocdump/main.go @@ -61,6 +61,7 @@ func main() { // timeout there silently drops the whole tree. Measured 2026-09-18. callTimeout := flag.Duration("timeout", 300*time.Second, "per LLM call timeout") parallel := flag.Int("parallel", 1, "documents in flight at once; the provider's adaptive limiter governs requests") + split := flag.Int("split", 0, "split leaves spanning more than this many pages into sub-leaves (0 = default 20, negative = off)") flag.Parse() if *parallel < 1 { *parallel = 1 @@ -119,7 +120,7 @@ func main() { if *judgeOnly { llm = refusingClient{} } - b := &ingest.TOCBuilder{LLM: llm, Judge: judge, LLMCallTimeout: *callTimeout, MinimalContext: *minimal} + b := &ingest.TOCBuilder{LLM: llm, Judge: judge, LLMCallTimeout: *callTimeout, MinimalContext: *minimal, SplitLeavesOver: *split} ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) start := time.Now() nodes, usage, err := b.Build(ctx, pages) diff --git a/docs/evaluations/2026-09-19-leaf-granularity.md b/docs/evaluations/2026-09-19-leaf-granularity.md new file mode 100644 index 0000000..cfab008 --- /dev/null +++ b/docs/evaluations/2026-09-19-leaf-granularity.md @@ -0,0 +1,80 @@ +# Leaf granularity — splitting the sections that hold the evidence + +**Date:** 2026-09-19 +**Harness:** [`cmd/tocdump -split`](../../cmd/tocdump/main.go), [`cmd/tocdump/coverage.py`](../../cmd/tocdump/coverage.py), [`cmd/tocdump/titles.py`](../../cmd/tocdump/titles.py), [`cmd/navbench`](../../cmd/navbench/main.go) +**Corpus:** FinanceBench, 21 10-K filings, 47 gold evidence pages, 40 questions +**Issue:** HAL-1374 +**Question:** a 10-K's tree is fine where nothing is (Items 1B–4, a paragraph each) and coarse where everything is (Item 8 and its notes, 70 pages under one title). Does splitting the big leaves at their own headings, on the Judge, make retrieval find more and cite tighter? + +## Result: right section 39 → 40 of 40; the leaf holding a gold page shrinks from 37 pages to 6 at the median; pages found stays at 36 of 40. Splitting defaults on at 20 pages. + +| tree | leaves / filing | span of the leaf holding a gold page, median | gold pages inside a leaf | right section | every gold page found | requests / question | $ / question | +|---|---|---|---|---|---|---|---| +| unsplit (contents-page grain) | 23 | **37 p** | 47 / 47 | 39 / 40 | 36 / 40 | 4.2 | 0.0035 | +| split over 8 | 69 | 1 p | 47 / 47 | **40 / 40** | 36 / 40 | 4.3 | 0.0041 | +| split over 12 | 67 | 1 p | 47 / 47 | **40 / 40** | 35 / 40 | 4.4 | 0.0041 | +| split over 20 | 68 | 1 p | 47 / 47 | **40 / 40** | 36 / 40 | 4.3 | 0.0040 | +| split over 12, tightened budget | 38 | 7 p | **39 / 47** | 33 / 40 | 30 / 40 | 4.2 | 0.0037 | +| **split over 20, final** | **54** | **6 p** | **47 / 47** | **40 / 40** | **36 / 40** | 4.3 | 0.0039 | + +All navigation rows use the navigator after the fix described below. +The TOC stage pays for the split: 271 Judge requests over the corpus +against 63, about $0.006 per filing against $0.0007, and 489 s wall at +parallel 8 against 122 s. Leaf titles against the unsplit tree: recall +0.971 (17 of 589 lost to sub-leaf re-titling), precision 0.542 — every +sub-leaf is an "extra" by construction. + +## What the sweep taught, in order + +1. **The first pass made navigation worse, and the tree was not at + fault.** At T=12 the right-section rate fell 37 → 33 and pages read + fell 40 → 16. The navigator took a fixed five sections; on a tree of + 80 one-page notes that was five pages. Sections are now taken in + rank order until the page budget is gathered. Same trees: right + section 33 → 40, evidence 31 → 35. (The bench's own `-leaves 3` + default masked the fix for one run.) +2. **Every threshold produced about the same leaf count: 69, 67 and 68 + per filing (medians) at 8, 12 and 20.** The per-page cap, not the + content, was deciding, and it kept the earliest headings rather than + the best. Sub-leaves are now kept by + the Judge's probability. +3. **Tightening the budget to one sub-leaf per half-threshold lost + coverage: 47 → 39 of 47.** Not because headings were dropped — because + sub-leaves began at the first confirmed heading, so the pages before + it (Item 8's index and the auditor's report) belonged to no leaf. A + gold page there was inside nothing. The section's opening is now its + first sub-leaf, titled as the parent. The loose budget was restored. +4. **8, 12 and 20 navigate alike.** 20 makes the fewest sub-leaves and + costs the least, so it is the default whenever a Judge is set. + +## Where the sub-leaves come from + +Two sources, in order of trust, both "select, don't generate": + +- **A nested contents page** inside the leaf — Item 8 opens with an + index of statements and notes with page numbers. Parsed, confirmed and + resolved exactly as the document's own contents page is. +- **Heading-shaped lines** — short, line-opening, Title Case or capitals, + not repeated on three or more pages (a running header is not a + heading). Each is judged with the lines that follow it: *is this a + heading a reader could turn to, rather than page furniture?* Boeing's + Item 7 became fifteen sub-sections from "Consolidated Results of + Operations" to "Contingent Obligations"; its Item 8 became the six + statements and twenty-one notes. + +## What is still missed + +Four questions, the same four as before the split: Boeing's legal +proceedings (the "see Note 21" hop now has a Note 21 leaf to reach, and +still misses the page inside it), and three page-level misses inside +the right section. Section choice is solved on this corpus; the +full-page question, or what it is shown, is next. + +## Reproduce + +```bash +go run ./cmd/tocdump -docs ~/.cache/vlbench/financebench -out /tmp/t -judge-only -minimal -parallel 8 # split on by default +go run ./cmd/tocdump -docs ~/.cache/vlbench/financebench -out /tmp/t0 -judge-only -minimal -parallel 8 -split -1 # off +python cmd/tocdump/coverage.py /tmp/t0 /tmp/t +go run ./cmd/navbench -questions ~/.cache/vlbench/financebench-questions.jsonl -trees /tmp/t -pdfs ~/.cache/vlbench/financebench -parallel 4 +``` diff --git a/pkg/ingest/toc_builder.go b/pkg/ingest/toc_builder.go index 83f8f34..2f404e6 100644 --- a/pkg/ingest/toc_builder.go +++ b/pkg/ingest/toc_builder.go @@ -105,6 +105,12 @@ type TOCBuilder struct { // to earn its place. MinimalContext bool + // SplitLeavesOver splits any leaf spanning more pages than this into + // sub-leaves at its internal headings, confirmed by the Judge + // (HAL-1374). Zero selects the default of 20 when a Judge is set; + // negative disables splitting. + SplitLeavesOver int + // DetectChars caps the characters of each page sent to detection when // MinimalContext is on. Zero means detectCharsMinimal. A contents page // declares itself in its first couple of thousand characters; the @@ -272,6 +278,15 @@ func (b *TOCBuilder) Build(ctx context.Context, pages []PageText) ([]tree.TOCNod // start pages drive the derivation. deriveEndPages(nodes, lastPage(pages)) + // Split the leaves that are too big to cite or to read, at their + // own internal headings (HAL-1374). Needs the spans, so it runs + // after end pages; adds its own children's end pages. + if over := b.splitLeavesOver(); over > 0 { + if n := b.splitLargeLeaves(ctx, nodes, pages, over, &usage); n > 0 { + log.Printf("toc: %d sub-leaves added inside leaves over %d pages", n, over) + } + } + // Stamp stable node IDs onto every node so callers / external // consumers have an opaque handle independent of position. stampNodeIDs(nodes, "") @@ -288,6 +303,21 @@ const resolverAttempts = 2 // exhaustion it leaves the tree exactly as extraction produced it and // records the fact; it never routes a Judge-path document through the // generative verifier. +// defaultSplitLeavesOver: measured on FinanceBench, splitting at 8, 12 +// and 20 pages navigated alike (right section 40/40, evidence 36/40); +// 20 makes the fewest sub-leaves and costs the least. +const defaultSplitLeavesOver = 20 + +func (b *TOCBuilder) splitLeavesOver() int { + switch { + case b.SplitLeavesOver < 0, b.Judge == nil: + return 0 + case b.SplitLeavesOver == 0: + return defaultSplitLeavesOver + } + return b.SplitLeavesOver +} + func (b *TOCBuilder) resolvePagesOrKeep(ctx context.Context, nodes []tree.TOCNode, pages []PageText, exclude []int, usage *Usage) { var lastErr error for attempt := 1; attempt <= resolverAttempts; attempt++ { diff --git a/pkg/ingest/toc_split.go b/pkg/ingest/toc_split.go new file mode 100644 index 0000000..6615215 --- /dev/null +++ b/pkg/ingest/toc_split.go @@ -0,0 +1,403 @@ +package ingest + +import ( + "context" + "fmt" + "log" + "regexp" + "sort" + "strings" + "unicode" + + "github.com/hallelx2/llmgate" + + "github.com/hallelx2/vectorless-engine/pkg/tree" +) + +// Leaf granularity (HAL-1374). +// +// A 10-K's tree is fine where nothing is and coarse where everything +// is: Items 1B, 2, 3 and 4 are a paragraph each, while Item 8 and its +// notes are 70 pages under one title. Retrieval pays for that — a +// 70-page leaf is a 40-page read to find one page — and a citation +// that says "Item 8" is not a citation. +// +// So a leaf whose span exceeds SplitLeavesOver is split into sub-leaves +// at its own internal headings, the same way the top level was built: +// code lists the candidates, the Judge confirms them. Two sources, in +// order of trust: +// +// 1. A nested contents page inside the leaf — Item 8 opens with an +// "Index to the Consolidated Financial Statements" listing every +// note with its page. Parsed, confirmed and resolved exactly like +// the document's own contents page. +// 2. Failing that, heading-shaped lines in the leaf's pages: short, +// line-opening, Title Case or capitals, not repeated on many pages +// (a running header is not a heading). The Judge sees each with the +// lines that follow it and says whether a sub-section starts there. + +const ( + // splitMinEntries is how many confirmed sub-headings make a split + // worth having. One is a title, not a structure. + splitMinEntries = 2 + // splitMaxPerPage bounds sub-leaves to one per two pages: finer than + // that is paragraphs, not sections. + splitMaxPerPage = 0.5 + // splitHeadingMaxWords bounds a heading-shaped line. + splitHeadingMaxWords = 9 + // splitRunningHeaderRepeats: a line seen on this many pages of one + // leaf is furniture. + splitRunningHeaderRepeats = 3 + // splitExcerptChars is the context shown with a candidate heading. + splitExcerptChars = 320 +) + +var ( + reSplitNoise = regexp.MustCompile(`(?i)^(table of contents|contents|page|index|continued|\(continued\)|notes? to (the )?consolidated financial statements|[\d,.$%()\s-]+)$`) + reSplitNumbered = regexp.MustCompile(`(?i)^(note|item|section|part)\s+\d+[a-c]?\b`) +) + +// splitLargeLeaves walks the tree and splits every leaf whose span +// exceeds over pages. It runs after end pages are derived, so spans are +// known, and re-derives them for the children it adds. Returns how many +// sub-leaves were added. +func (b *TOCBuilder) splitLargeLeaves(ctx context.Context, nodes []tree.TOCNode, pages []PageText, over int, usage *Usage) int { + if over <= 0 || b.Judge == nil { + return 0 + } + byPage := map[int]string{} + for _, p := range pages { + byPage[p.PageNumber] = p.Text + } + added := 0 + var walk func(ns []tree.TOCNode) + walk = func(ns []tree.TOCNode) { + for i := range ns { + n := &ns[i] + if len(n.Nodes) > 0 { + walk(n.Nodes) + continue + } + if n.StartPage <= 0 || n.EndPage < n.StartPage || n.EndPage-n.StartPage+1 <= over { + continue + } + subs, err := b.splitLeaf(ctx, n, pages, byPage, over, usage) + if err != nil { + log.Printf("toc: split %q failed, leaf kept whole: %v", n.Title, err) + usage.degrade("leaf split", fmt.Sprintf("%q kept whole: %v", n.Title, err)) + continue + } + if len(subs) < splitMinEntries { + continue + } + // The pages before the first heading are the section's own + // opening — Item 8's index and auditor's report before the + // first statement. They must belong to a leaf, or a gold page + // there is inside no leaf at all (measured: 47/47 → 39/47). + if subs[0].StartPage > n.StartPage { + subs = append([]tree.TOCNode{{Title: n.Title, StartPage: n.StartPage}}, subs...) + } + for j := range subs { + subs[j].Structure = fmt.Sprintf("%s.%d", n.Structure, j+1) + } + deriveEndPagesIn(subs, n.EndPage) + n.Nodes = subs + added += len(subs) + } + } + walk(nodes) + return added +} + +// splitLeaf returns the sub-leaves of one leaf, with start pages, in +// page order. +func (b *TOCBuilder) splitLeaf(ctx context.Context, leaf *tree.TOCNode, pages []PageText, byPage map[int]string, over int, usage *Usage) ([]tree.TOCNode, error) { + var leafPages []PageText + for _, p := range pages { + if p.PageNumber >= leaf.StartPage && p.PageNumber <= leaf.EndPage { + leafPages = append(leafPages, p) + } + } + if len(leafPages) < 2 { + return nil, nil + } + + // 1. A nested contents page. + if idx, entries := nestedContents(leafPages); idx > 0 { + subs, err := b.subLeavesFromContents(ctx, leaf, entries, byPage[idx], leafPages, idx, usage) + if err != nil { + return nil, err + } + if len(subs) >= splitMinEntries { + return subs, nil + } + } + + // 2. Heading-shaped lines. + cands := headingCandidates(leafPages) + if len(cands) < splitMinEntries { + return nil, nil + } + return b.subLeavesFromHeadings(ctx, leaf, cands, over, usage) +} + +// nestedContents finds a page inside the leaf that reads as a table of +// contents with at least three entries, and returns its number and +// entries. Zero when there is none. +func nestedContents(leafPages []PageText) (int, []contentsEntry) { + for _, p := range leafPages { + if !prefilterTOC(p.Text).Any() { + continue + } + es := parseContentsEntries(p.Text) + if countRealEntries(es) >= 3 { + return p.PageNumber, es + } + } + return 0, nil +} + +// subLeavesFromContents confirms the nested contents' entries and +// resolves each within the leaf's pages, exactly as the top level does. +func (b *TOCBuilder) subLeavesFromContents(ctx context.Context, leaf *tree.TOCNode, entries []contentsEntry, contents string, leafPages []PageText, indexPage int, usage *Usage) ([]tree.TOCNode, error) { + keep, err := b.confirmEntriesJudge(ctx, entries, contents, usage) + if err != nil { + return nil, err + } + var subs []tree.TOCNode + for i, e := range entries { + if !keep[i] || e.Container || normalise(e.Title) == normalise(leaf.Title) { + continue // the leaf's own title is not a sub-section of it + } + subs = append(subs, tree.TOCNode{Title: e.Title}) + } + if len(subs) < splitMinEntries { + return nil, nil + } + resolved, handled, err := b.resolvePagesJudgeErr(ctx, subs, leafPages, []int{indexPage}, usage) + if err != nil { + return nil, err + } + if handled { + applyResolvedPages(subs, resolved) + } + // Keep what was placed, in page order; a sub-section that could not + // be found in the leaf's pages does not become a leaf that points + // nowhere. + var placed []tree.TOCNode + for _, s := range subs { + if s.StartPage >= leaf.StartPage && s.StartPage <= leaf.EndPage { + placed = append(placed, s) + } + } + sortByStart(placed) + return capSubLeaves(placed, leaf), nil +} + +// headingCandidate is a line that looks like a heading, with where it is. +type headingCandidate struct { + text string + page int + excerpt string +} + +// headingCandidates lists the heading-shaped lines of a leaf's pages, +// every occurrence, dropping lines that repeat on three or more pages +// (a running header). A title that first appears in a mini-index or a +// cross-reference and later opens the real sub-section is judged at +// each place it appears; accepted duplicates are resolved afterwards. +func headingCandidates(leafPages []PageText) []headingCandidate { + seenOnPages := map[string]map[int]bool{} + var all []headingCandidate + for _, p := range leafPages { + lines := strings.Split(p.Text, "\n") + for i, raw := range lines { + line := strings.TrimSpace(strings.Trim(raw, "* \t")) + if !looksLikeSubHeading(line) { + continue + } + key := normalise(line) + if seenOnPages[key] == nil { + seenOnPages[key] = map[int]bool{} + } + seenOnPages[key][p.PageNumber] = true + rest := strings.Join(lines[i+1:], "\n") + if len(rest) > splitExcerptChars { + rest = rest[:splitExcerptChars] + } + all = append(all, headingCandidate{text: line, page: p.PageNumber, excerpt: line + "\n" + rest}) + } + } + var out []headingCandidate + for _, c := range all { + if len(seenOnPages[normalise(c.text)]) >= splitRunningHeaderRepeats { + continue + } + out = append(out, c) + } + return out +} + +// looksLikeSubHeading is the code-side filter: short, opens with a +// capital, Title Case or all capitals, no sentence punctuation, not +// page furniture. The Judge decides the rest. +func looksLikeSubHeading(line string) bool { + if line == "" || len(line) > 90 { + return false + } + if reSplitNoise.MatchString(line) { + return false + } + if reSplitNumbered.MatchString(line) { + return true + } + words := strings.Fields(line) + if len(words) == 0 || len(words) > splitHeadingMaxWords { + return false + } + if strings.HasSuffix(line, ".") || strings.HasSuffix(line, ",") || strings.HasSuffix(line, ";") || strings.HasSuffix(line, ":") { + return false + } + r := []rune(words[0])[0] + if !unicode.IsUpper(r) { + return false + } + caps, lower := 0, 0 + for _, w := range words { + if isStopword(strings.ToLower(w)) { + continue + } + fr := []rune(w)[0] + if unicode.IsUpper(fr) { + caps++ + } else if unicode.IsLetter(fr) { + lower++ + } + } + return caps > 0 && lower == 0 +} + +// subLeavesFromHeadings asks the Judge, for each candidate, whether a +// sub-section of the leaf begins at that line. +func (b *TOCBuilder) subLeavesFromHeadings(ctx context.Context, leaf *tree.TOCNode, cands []headingCandidate, over int, usage *Usage) ([]tree.TOCNode, error) { + th := b.judgeThreshold() + prob := make([]float64, len(cands)) + const perBatch = 80 + for start := 0; start < len(cands); start += perBatch { + end := start + perBatch + if end > len(cands) { + end = len(cands) + } + state := map[string]any{"section": leaf.Title} + questions := map[string]llmgate.Question{} + for i := start; i < end; i++ { + qk := fmt.Sprintf("h_%d", i) + state[qk] = map[string]any{"line": cands[i].text, "passage": cands[i].excerpt} + questions[qk] = llmgate.Noul{ + Instructions: fmt.Sprintf( + "`section` is the title of a long section of a document. `%s.passage` is a "+ + "passage from inside it, beginning with the line `%s.line`. Is that line a "+ + "heading that opens a sub-section a reader could turn to — a note, a topic, a "+ + "statement — rather than a running header, a table's column label, a caption, "+ + "or a sentence fragment?", qk, qk), + Criteria: &llmgate.NoulCriteria{ + True: "The line is a sub-section heading and its content follows it", + False: "The line is page furniture, a label inside a table, or not a heading", + }, + } + } + res, err := b.Judge.Judge(ctx, llmgate.JudgeRequest{State: state, Questions: questions}) + if err != nil { + return nil, err + } + addJudgeUsage(usage, res) + for qk := range questions { + p, err := res.Noul(qk) + if err != nil { + continue + } + var i int + fmt.Sscanf(qk, "h_%d", &i) + prob[i] = p + } + } + // Headings found without an index are the weaker source, so the + // budget is tighter: about one sub-leaf per half-threshold of + // pages, the most confident kept, then put back in page order. + // Without this every threshold produced the same 68 leaves per + // filing — the per-page cap, not the content, was deciding. + type scored struct { + n tree.TOCNode + p float64 + } + var ss []scored + for i, c := range cands { + if prob[i] > th && normalise(c.text) != normalise(leaf.Title) { + ss = append(ss, scored{tree.TOCNode{Title: c.text, StartPage: c.page}, prob[i]}) + } + } + // Measured (HAL-1374): a budget of one sub-leaf per half-threshold + // dropped real headings and cost coverage; one per two pages, the + // most confident kept, matched the index path and navigated best. + span := leaf.EndPage - leaf.StartPage + 1 + max := int(float64(span) * splitMaxPerPage) + if max < splitMinEntries { + max = splitMinEntries + } + _ = over + sort.SliceStable(ss, func(i, j int) bool { return ss[i].p > ss[j].p }) + // One sub-leaf per page and per title, the most confident of each, + // chosen before the cap so a page with several confident headings + // cannot spend the whole budget. + var subs []tree.TOCNode + seenPage := map[int]bool{} + seenTitle := map[string]bool{} + for _, x := range ss { + if seenPage[x.n.StartPage] || seenTitle[normalise(x.n.Title)] { + continue + } + seenPage[x.n.StartPage] = true + seenTitle[normalise(x.n.Title)] = true + subs = append(subs, x.n) + if len(subs) >= max { + break + } + } + sortByStart(subs) + return subs, nil +} + +func sortByStart(ns []tree.TOCNode) { + for i := 1; i < len(ns); i++ { + for j := i; j > 0 && ns[j].StartPage < ns[j-1].StartPage; j-- { + ns[j], ns[j-1] = ns[j-1], ns[j] + } + } +} + +// capSubLeaves keeps at most one sub-leaf per two pages of the parent, +// dropping later entries that share a page with an earlier one first. +func capSubLeaves(subs []tree.TOCNode, leaf *tree.TOCNode) []tree.TOCNode { + span := leaf.EndPage - leaf.StartPage + 1 + max := int(float64(span) * splitMaxPerPage) + if max < splitMinEntries { + max = splitMinEntries + } + if len(subs) <= max { + return subs + } + // Prefer one per page: drop same-page repeats, then truncate. + var out []tree.TOCNode + lastPage := -1 + for _, s := range subs { + if s.StartPage == lastPage { + continue + } + out = append(out, s) + lastPage = s.StartPage + } + if len(out) > max { + out = out[:max] + } + return out +} diff --git a/pkg/ingest/toc_split_test.go b/pkg/ingest/toc_split_test.go new file mode 100644 index 0000000..b041e37 --- /dev/null +++ b/pkg/ingest/toc_split_test.go @@ -0,0 +1,257 @@ +package ingest + +import ( + "context" + "strings" + "testing" + + "github.com/hallelx2/llmgate" + + "github.com/hallelx2/vectorless-engine/pkg/tree" +) + +// splitJudge confirms contents entries and heading lines whose text +// contains any of the given words, resolves a title to the page whose +// excerpt opens with it, and rejects everything else. +func splitJudge(yes ...string) *llmgate.MockJudge { + return &llmgate.MockJudge{Respond: func(_ context.Context, req llmgate.JudgeRequest) (*llmgate.Judgment, error) { + st := req.State.(map[string]any) + ans := map[string]llmgate.Answer{} + for id := range req.Questions { + item, _ := st[id].(map[string]any) + p := 0.05 + switch { + case strings.HasPrefix(id, "e_"), strings.HasPrefix(id, "h_"): + text, _ := item["title"].(string) + if text == "" { + text, _ = item["line"].(string) + } + for _, w := range yes { + if strings.Contains(strings.ToLower(text), w) { + p = 0.9 + } + } + case strings.HasPrefix(id, "r_"): + title, _ := item["title"].(string) + ex, _ := item["excerpt"].(string) + if strings.HasPrefix(strings.ToLower(ex), strings.ToLower(title)) { + p = 0.9 + } + } + ans[id] = llmgate.NoulAnswer{Noul: p} + } + return &llmgate.Judgment{Model: "mock", Answers: ans, Usage: llmgate.Usage{TokensReported: true}}, nil + }} +} + +func item8Pages() []PageText { + ps := []PageText{ + {54, "Item 8. Financial Statements and Supplementary Data\nIndex to the Consolidated Financial Statements\nPage\nConsolidated Statements of Operations 56\nConsolidated Statements of Financial Position 57\nNote 1 - Summary of Significant Accounting Policies 59\nNote 2 - Goodwill and Acquired Intangibles 70\nNote 21 - Legal Proceedings 113"}, + {55, "Table of Contents\nReport of Independent Registered Public Accounting Firm\nWe have audited the accompanying statements."}, + {56, "Table of Contents\nConsolidated Statements of Operations\nRevenues 66,608"}, + {57, "Table of Contents\nConsolidated Statements of Financial Position\nAssets\nCash 14,614"}, + } + for p := 58; p <= 125; p++ { + text := "Table of Contents\nNotes to the Consolidated Financial Statements\nprose about accounting " + strings.Repeat("x ", 50) + switch p { + case 59: + text = "Table of Contents\nNote 1 - Summary of Significant Accounting Policies\nPrinciples of Consolidation and Basis of Presentation\nThe consolidated financial statements included." + case 70: + text = "Table of Contents\nNote 2 - Goodwill and Acquired Intangibles\nGoodwill is tested annually." + case 113: + text = "Table of Contents\nNote 21 - Legal Proceedings\nA class action filed in 2019 remains pending." + } + ps = append(ps, PageText{p, text}) + } + return ps +} + +func TestSplitLeafFromANestedContentsPage(t *testing.T) { + b := &TOCBuilder{Judge: splitJudge("note", "consolidated statements")} + nodes := []tree.TOCNode{{Structure: "2", Title: "PART II", StartPage: 35, EndPage: 125, Nodes: []tree.TOCNode{ + {Structure: "2.4", Title: "Item 7A. Quantitative and Qualitative Disclosures", StartPage: 53, EndPage: 53}, + {Structure: "2.5", Title: "Item 8. Financial Statements and Supplementary Data", StartPage: 54, EndPage: 125}, + }}} + var usage Usage + n := b.splitLargeLeaves(context.Background(), nodes, item8Pages(), 12, &usage) + if n < 3 { + t.Fatalf("sub-leaves added: %d, want the statements and notes", n) + } + item8 := nodes[0].Nodes[1] + if len(item8.Nodes) != n { + t.Fatalf("children on Item 8: %d", len(item8.Nodes)) + } + byTitle := map[string]tree.TOCNode{} + for _, c := range item8.Nodes { + byTitle[c.Title] = c + } + n21, ok := byTitle["Note 21 - Legal Proceedings"] + if !ok || n21.StartPage != 113 { + t.Errorf("Note 21 should be a sub-leaf on page 113: %+v (all: %v)", n21, titlesOfNodes(item8.Nodes)) + } + if n1 := byTitle["Note 1 - Summary of Significant Accounting Policies"]; n1.StartPage != 59 || n1.EndPage != 69 { + t.Errorf("Note 1 span: %d-%d want 59-69", n1.StartPage, n1.EndPage) + } + if last := item8.Nodes[len(item8.Nodes)-1]; last.EndPage != 125 { + t.Errorf("last sub-leaf should run to the parent's end: %d", last.EndPage) + } + if !strings.HasPrefix(item8.Nodes[0].Structure, "2.5.") { + t.Errorf("sub-leaf structure: %q", item8.Nodes[0].Structure) + } + // The small leaf beside it is untouched. + if len(nodes[0].Nodes[0].Nodes) != 0 { + t.Errorf("a one-page leaf was split") + } + if len(usage.Degraded) != 0 { + t.Errorf("degraded: %v", usage.Degraded) + } + // The pages before the first statement (index, auditor's report) + // belong to an opening sub-leaf with the parent's title. + if first := item8.Nodes[0]; first.StartPage != 54 || first.Title != item8.Title { + t.Errorf("opening sub-leaf should cover the parent's head from page 54: %+v", first) + } +} + +func TestSplitLeafFromHeadingLinesWhenThereIsNoIndex(t *testing.T) { + var ps []PageText + for p := 22; p <= 52; p++ { + text := "Table of Contents\nprose about results " + strings.Repeat("y ", 40) + switch p { + case 22: + text = "Item 7. Management’s Discussion and Analysis\nBusiness Environment and Trends\nThe commercial aviation market continued to recover." + case 30: + text = "Table of Contents\nResults of Operations\nRevenues\nThe following table summarizes." + case 41: + text = "Table of Contents\nLiquidity and Capital Resources\nWe believe our cash position is adequate." + case 45: + text = "Table of Contents\nCritical Accounting Estimates\nProgram Accounting\nManagement uses estimates." + } + ps = append(ps, PageText{p, text}) + } + b := &TOCBuilder{Judge: splitJudge("business environment", "results of operations", "liquidity", "critical accounting")} + nodes := []tree.TOCNode{{Structure: "2.3", Title: "Item 7. Management’s Discussion and Analysis", StartPage: 22, EndPage: 52}} + var usage Usage + n := b.splitLargeLeaves(context.Background(), nodes, ps, 12, &usage) + if n != 4 { + t.Fatalf("sub-leaves: %d want 4: %v", n, titlesOfNodes(nodes[0].Nodes)) + } + got := titlesOfNodes(nodes[0].Nodes) + want := "Business Environment and Trends|Results of Operations|Liquidity and Capital Resources|Critical Accounting Estimates" + if strings.Join(got, "|") != want { + t.Errorf("got %v", got) + } + if nodes[0].Nodes[1].StartPage != 30 || nodes[0].Nodes[1].EndPage != 40 { + t.Errorf("Results of Operations span %d-%d want 30-40", nodes[0].Nodes[1].StartPage, nodes[0].Nodes[1].EndPage) + } +} + +func TestHeadingCandidatesDropRunningHeadersAndFurniture(t *testing.T) { + ps := []PageText{ + {1, "Table of Contents\nThe Boeing Company and Subsidiaries\nRevenue and Related Cost Recognition\nWe recognize revenue when."}, + {2, "Table of Contents\nThe Boeing Company and Subsidiaries\nAssets\n14,614\nUse of Estimates\nEstimates are used."}, + {3, "Table of Contents\nThe Boeing Company and Subsidiaries\nOperating Cycle\nOur operating cycle is long."}, + } + got := headingCandidates(ps) + titles := []string{} + for _, c := range got { + titles = append(titles, c.text) + } + joined := strings.Join(titles, "|") + for _, bad := range []string{"Table of Contents", "The Boeing Company and Subsidiaries", "14,614"} { + if strings.Contains(joined, bad) { + t.Errorf("furniture admitted: %q in %v", bad, titles) + } + } + for _, good := range []string{"Revenue and Related Cost Recognition", "Use of Estimates", "Operating Cycle", "Assets"} { + if !strings.Contains(joined, good) { + t.Errorf("heading missed: %q in %v", good, titles) + } + } + if !strings.HasPrefix(got[0].excerpt, "Revenue and Related Cost Recognition\nWe recognize") { + t.Errorf("excerpt should start at the heading: %q", got[0].excerpt) + } +} + +// A title that appears first in a two-entry mini-index and later as the +// real heading is judged at both places, and the accepted one wins. +func TestHeadingCandidatesKeepEveryOccurrence(t *testing.T) { + ps := []PageText{ + {60, "Contents of this note\nRevenue Recognition\nLeases\nsee the sections below"}, + {64, "Revenue Recognition\nWe recognize revenue when control transfers."}, + } + got := headingCandidates(ps) + n := 0 + for _, c := range got { + if c.text == "Revenue Recognition" { + n++ + } + } + if n != 2 { + t.Errorf("want both occurrences judged, got %d", n) + } +} + +// Several confident headings on one page cannot spend the whole cap. +func TestSubLeavesOnePerPageBeforeTheCap(t *testing.T) { + b := &TOCBuilder{Judge: splitJudge("alpha", "beta", "gamma", "delta")} + leaf := &tree.TOCNode{Title: "Big", StartPage: 1, EndPage: 4} + cands := []headingCandidate{ + {text: "Alpha One", page: 1, excerpt: "Alpha One\nx"}, {text: "Alpha Two", page: 1, excerpt: "Alpha Two\nx"}, {text: "Alpha Three", page: 1, excerpt: "Alpha Three\nx"}, + {text: "Beta", page: 2, excerpt: "Beta\nx"}, {text: "Gamma", page: 3, excerpt: "Gamma\nx"}, {text: "Delta", page: 4, excerpt: "Delta\nx"}, + } + var usage Usage + subs, err := b.subLeavesFromHeadings(context.Background(), leaf, cands, 20, &usage) + if err != nil { + t.Fatal(err) + } + pages := map[int]bool{} + for _, s := range subs { + pages[s.StartPage] = true + } + if len(subs) != 2 || len(pages) != 2 { + t.Errorf("a 4-page leaf caps at 2 sub-leaves on 2 distinct pages, got %v", titlesOfNodes(subs)) + } +} + +func TestLooksLikeSubHeading(t *testing.T) { + yes := []string{"Note 21 - Legal Proceedings", "Revenue and Related Cost Recognition", "CONSOLIDATED BALANCE SHEET", "Use of Estimates", "Item 1A. Risk Factors"} + no := []string{"We recognize revenue when control transfers.", "Table of Contents", "14,614", "Page", "the following table summarizes", "Revenues:", "Total Amounts Paid To Each Of The Named Executive Officers During The Year Ended"} + for _, s := range yes { + if !looksLikeSubHeading(s) { + t.Errorf("%q should look like a heading", s) + } + } + for _, s := range no { + if looksLikeSubHeading(s) { + t.Errorf("%q should not look like a heading", s) + } + } +} + +func TestSplitIsOffWithoutAJudgeOrAThreshold(t *testing.T) { + nodes := []tree.TOCNode{{Title: "Item 8", StartPage: 54, EndPage: 125}} + var usage Usage + if n := (&TOCBuilder{}).splitLargeLeaves(context.Background(), nodes, item8Pages(), 12, &usage); n != 0 { + t.Errorf("no Judge: %d", n) + } + if n := (&TOCBuilder{Judge: splitJudge("note")}).splitLargeLeaves(context.Background(), nodes, item8Pages(), 0, &usage); n != 0 { + t.Errorf("threshold 0 means off: %d", n) + } + if got := (&TOCBuilder{Judge: splitJudge("x"), SplitLeavesOver: -1}).splitLeavesOver(); got != 0 { + t.Errorf("negative should disable: %d", got) + } + if got := (&TOCBuilder{Judge: splitJudge("x")}).splitLeavesOver(); got != defaultSplitLeavesOver { + t.Errorf("default with a Judge: %d", got) + } + if got := (&TOCBuilder{}).splitLeavesOver(); got != 0 { + t.Errorf("no Judge means off: %d", got) + } +} + +func titlesOfNodes(ns []tree.TOCNode) []string { + out := []string{} + for _, n := range ns { + out = append(out, n.Title) + } + return out +} diff --git a/pkg/retrieval/judgewalk.go b/pkg/retrieval/judgewalk.go index 1744122..a8be4f8 100644 --- a/pkg/retrieval/judgewalk.go +++ b/pkg/retrieval/judgewalk.go @@ -77,7 +77,10 @@ type JudgeNavigator struct { Threshold float64 // MaxLeaves bounds how many sections' pages are gathered. Zero - // selects 5. + // means the page budget alone decides: sections are taken in rank + // order until CoarsePages are gathered. A fixed count of five was + // right for a 23-leaf 10-K and wrong for one split into 80 — + // five one-page notes and the budget went unused (HAL-1374). MaxLeaves int // MaxPages bounds how many pages are judged in full. Zero selects 40. @@ -115,7 +118,6 @@ type JudgeNavigator struct { const ( defaultNavThreshold = 0.5 - defaultNavMaxLeaves = 5 defaultNavMaxPages = 40 defaultNavCoarse = 120 defaultNavHeadChars = 700 @@ -137,7 +139,7 @@ func (n *JudgeNavigator) maxLeaves() int { if n.MaxLeaves > 0 { return n.MaxLeaves } - return defaultNavMaxLeaves + return 1 << 30 // the page budget governs } func (n *JudgeNavigator) maxPages() int { diff --git a/pkg/retrieval/judgewalk_test.go b/pkg/retrieval/judgewalk_test.go index a6fab82..12552ce 100644 --- a/pkg/retrieval/judgewalk_test.go +++ b/pkg/retrieval/judgewalk_test.go @@ -3,6 +3,7 @@ package retrieval import ( "context" "errors" + "fmt" "strings" "sync/atomic" "testing" @@ -284,3 +285,27 @@ func TestReferencedLeaves(t *testing.T) { t.Errorf("got %v want [b c a] — and Note 2 must not match Note 21, Item 1 must not match Item 1A", ids) } } + +// On a finely split tree, sections are taken in rank order until the +// page budget is full — not a fixed five. +func TestNavigateFillsThePageBudgetOnAFineTree(t *testing.T) { + j, _ := navJudge("note", "needle") + n := &JudgeNavigator{Judge: j, MaxPages: 10, CoarsePages: 30} + var leaves []NavLeaf + for i := 1; i <= 40; i++ { + leaves = append(leaves, NavLeaf{ID: fmt.Sprint(i), Title: fmt.Sprintf("Note %d - Topic", i), Start: 60 + i, End: 60 + i}) + } + load := func(_ context.Context, l NavLeaf) ([]NavPage, error) { + return []NavPage{{Number: l.Start, Text: "prose"}}, nil + } + res, err := n.Navigate(context.Background(), "q", leaves, load) + if err != nil { + t.Fatal(err) + } + if len(res.Selected) != 30 { + t.Errorf("one-page leaves should be gathered up to the coarse budget of 30, got %d", len(res.Selected)) + } + if len(res.Pages) != 10 { + t.Errorf("full read should be MaxPages=10, got %d", len(res.Pages)) + } +}