diff --git a/cmd/engine/main.go b/cmd/engine/main.go index dc126d6..74a22b4 100644 --- a/cmd/engine/main.go +++ b/cmd/engine/main.go @@ -7,6 +7,7 @@ package main import ( "context" "crypto/tls" + "encoding/json" "errors" "flag" "fmt" @@ -36,6 +37,7 @@ import ( "github.com/hallelx2/vectorless-engine/pkg/queue" "github.com/hallelx2/vectorless-engine/pkg/retrieval" "github.com/hallelx2/vectorless-engine/pkg/storage" + "github.com/hallelx2/vectorless-engine/pkg/tree" ) // version is set at build time via -ldflags "-X main.version=..." @@ -141,7 +143,7 @@ func run() error { } else { logger.Warn("judge: none configured — TOC judgements run on the generative driver, one call per page (set TYPESAFE_API_KEY)") } - strategy := buildStrategy(cfg.Retrieval, llmClient, judge, store) + strategy := buildStrategy(cfg.Retrieval, llmClient, judge, store, pool) // Wrap with caching if enabled. if cfg.Retrieval.Cache.Enabled { @@ -249,6 +251,8 @@ func run() error { }) if cfg.Ingest.Mode == ingest.ModeMinimal { logger.Info("ingest: MINIMAL mode — parse→persist→ready; skipping summarize/HyDE/multi-axis/TOC + table extraction") + } else if cfg.Ingest.Mode == ingest.ModeTOC { + logger.Info("ingest: TOC mode — parse→table of contents→persist→ready; skipping summarize/HyDE/multi-axis + table extraction") } else if cfg.Ingest.Tables.Enabled { logger.Info("ingest: pdf table extraction enabled", "vertical_strategy", cfg.Ingest.Tables.VerticalStrategy, @@ -540,7 +544,7 @@ func buildLLMFrom(c config.LLMConfig, provider, apiKey, baseURL, model string) ( } } -func buildStrategy(c config.RetrievalConfig, client llmgate.Client, judge llmgate.Judge, store storage.Storage) retrieval.Strategy { +func buildStrategy(c config.RetrievalConfig, client llmgate.Client, judge llmgate.Judge, store storage.Storage, pool *db.Pool) retrieval.Strategy { switch c.Strategy { case "judgewalk": if judge == nil { @@ -549,6 +553,8 @@ func buildStrategy(c config.RetrievalConfig, client llmgate.Client, judge llmgat } s := retrieval.NewJudgeWalkStrategy(judge) s.PageLoader = storagePageLoader{s: store} + s.TOC = dbTOCProvider{db: pool} + s.Pages = storagePageStore{s: store} return s case "single-pass": return retrieval.NewSinglePass(client) @@ -680,3 +686,39 @@ func tableOptsFromConfig(c config.TablesConfig) *parser.TableOpts { MinTableCols: c.MinTableCols, } } + +// storagePageStore serves the per-page text ingest persisted at +// ingest.PagesKey, for judgewalk. Missing pages are not an error: the +// strategy falls back to the section tree. +type storagePageStore struct{ s storage.Storage } + +func (p storagePageStore) LoadPages(ctx context.Context, docID tree.DocumentID) ([]retrieval.NavPage, error) { + rc, _, err := p.s.Get(ctx, ingest.PagesKey(docID)) + if err != nil { + return nil, err + } + defer rc.Close() + var pages []ingest.PageText + if err := json.NewDecoder(rc).Decode(&pages); err != nil { + return nil, err + } + out := make([]retrieval.NavPage, 0, len(pages)) + for _, pg := range pages { + out = append(out, retrieval.NavPage{Number: pg.PageNumber, Text: pg.Text}) + } + return out, nil +} + +// dbTOCProvider reads documents.toc_tree for judgewalk and treewalk. +type dbTOCProvider struct{ db *db.Pool } + +func (p dbTOCProvider) GetTOC(ctx context.Context, docID tree.DocumentID) ([]byte, error) { + doc, err := p.db.GetDocumentForWorker(ctx, docID) + if err != nil { + return nil, err + } + if len(doc.TOCTree) == 0 { + return nil, retrieval.ErrNoTOC + } + return doc.TOCTree, nil +} diff --git a/cmd/navbench/main.go b/cmd/navbench/main.go index e8f541c..b07bc12 100644 --- a/cmd/navbench/main.go +++ b/cmd/navbench/main.go @@ -87,7 +87,12 @@ func main() { fmt.Fprintln(os.Stderr, "judge:", err) os.Exit(1) } - lim := limit.New(limit.Config{Initial: 4, OnChange: func(e limit.Event) { + // Initial 16, not 4. A question issues 4-7 requests that do not + // depend on each other, so a limiter that starts at 4 serialises + // them into waves, and AIMD needs twenty successes to widen by one — + // slower than a 40-question run can recover from, especially after a + // transient failure halves it. + lim := limit.New(limit.Config{Initial: 16, OnChange: func(e limit.Event) { fmt.Fprintf(os.Stderr, " limiter %s %d -> %d %v\n", e.Cause, e.From, e.To, e.Err) }}) nav := &retrieval.JudgeNavigator{Judge: retry.NewJudge(retry.Config{MaxRetries: 3})(limit.Judge(lim)(tj)), MaxLeaves: *maxLeaves, MaxPages: *maxPages} diff --git a/cmd/server/main.go b/cmd/server/main.go index 48fe422..b6f1728 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -15,6 +15,7 @@ package main import ( "context" "crypto/tls" + "encoding/json" "errors" "flag" "fmt" @@ -241,6 +242,8 @@ func run() error { }) if cfg.Engine.Ingest.Mode == ingest.ModeMinimal { logger.Info("ingest: MINIMAL mode — parse→persist→ready; skipping summarize/HyDE/multi-axis/TOC + table extraction") + } else if cfg.Engine.Ingest.Mode == ingest.ModeTOC { + logger.Info("ingest: TOC mode — parse→table of contents→persist→ready; skipping summarize/HyDE/multi-axis + table extraction") } else if cfg.Engine.Ingest.Tables.Enabled { logger.Info("ingest: pdf table extraction enabled", "vertical_strategy", cfg.Engine.Ingest.Tables.VerticalStrategy, @@ -488,7 +491,7 @@ func buildStrategy(c enginecfg.RetrievalConfig, client llmgate.Client, judge llm log.Printf("retrieval: strategy judgewalk needs llm.judge configured; using treewalk") return buildTreeWalkStrategy(c, client, store, pool) } - return buildJudgeWalkStrategy(judge, store) + return buildJudgeWalkStrategy(judge, store, pool) case "single-pass": return retrieval.NewSinglePass(client) case "chunked-tree": @@ -535,7 +538,12 @@ func buildStrategySet(c enginecfg.RetrievalConfig, client llmgate.Client, judge "auto": retrieval.NewAuto(retrieval.NewSinglePass(client), buildTreeWalkStrategy(c, client, store, pool)), } if judge != nil { - set["judgewalk"] = buildJudgeWalkStrategy(judge, store) + set["judgewalk"] = buildJudgeWalkStrategy(judge, store, pool) + } else { + // The same fallback the default builder applies: a request that + // names judgewalk on a server with no Judge gets treewalk, not + // "unknown strategy". + set["judgewalk"] = set["treewalk"] } return set } @@ -544,9 +552,13 @@ func buildStrategySet(c enginecfg.RetrievalConfig, client llmgate.Client, judge // leaves ranked in one request, the best sections' bodies ranked in one // or two more, no generative call (HAL-1371). Selectable per request as // strategy=judgewalk whenever llm.judge is configured. -func buildJudgeWalkStrategy(judge llmgate.Judge, store storage.Storage) *retrieval.JudgeWalkStrategy { +func buildJudgeWalkStrategy(judge llmgate.Judge, store storage.Storage, pool *db.Pool) *retrieval.JudgeWalkStrategy { s := retrieval.NewJudgeWalkStrategy(judge) s.PageLoader = storagePageLoader{s: store} + if pool != nil { + s.TOC = dbTOCProvider{db: pool} + } + s.Pages = storagePageStore{s: store} return s } @@ -687,3 +699,25 @@ func tableOptsFromConfig(c enginecfg.TablesConfig) *parser.TableOpts { MinTableCols: c.MinTableCols, } } + +// storagePageStore serves the per-page text ingest persisted at +// ingest.PagesKey, for judgewalk. Missing pages are not an error: the +// strategy falls back to the section tree. +type storagePageStore struct{ s storage.Storage } + +func (p storagePageStore) LoadPages(ctx context.Context, docID tree.DocumentID) ([]retrieval.NavPage, error) { + rc, _, err := p.s.Get(ctx, ingest.PagesKey(docID)) + if err != nil { + return nil, err + } + defer rc.Close() + var pages []ingest.PageText + if err := json.NewDecoder(rc).Decode(&pages); err != nil { + return nil, err + } + out := make([]retrieval.NavPage, 0, len(pages)) + for _, pg := range pages { + out = append(out, retrieval.NavPage{Number: pg.PageNumber, Text: pg.Text}) + } + return out, nil +} diff --git a/config.example.yaml b/config.example.yaml index be9e3f3..5336fd6 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -362,6 +362,13 @@ ingest: # summary-dependent strategies (chunked-tree, agentic) # degrade to titles + raw content with no summaries. # + # toc parse -> build tree -> table of contents -> persist -> + # ready. The page-based pipeline and nothing else: on a + # Judge the TOC stage is three requests and seconds, and + # page-based retrieval (treewalk, judgewalk) needs none of + # the per-section enrichment. Table extraction skipped as + # in minimal. The mode the FinanceBench evaluations use. + # # Override per-process with VLE_INGEST_MODE; on the deployed # vectorless-server use VLS_INGEST_MODE=minimal (no secret edit needed). mode: "full" diff --git a/docs/evaluations/2026-09-19-head-to-head-chunk-and-embed.md b/docs/evaluations/2026-09-19-head-to-head-chunk-and-embed.md new file mode 100644 index 0000000..1ebf425 --- /dev/null +++ b/docs/evaluations/2026-09-19-head-to-head-chunk-and-embed.md @@ -0,0 +1,91 @@ +# Vectorless on Jev against chunk-and-embed, on FinanceBench + +**Date:** 2026-09-19 +**Harness:** [vectorless-bench](https://github.com/hallelx2/vectorless-bench) `configs/financebench_jev.yaml` — the engine reached through the Python SDK, the baselines in-process +**Corpus:** FinanceBench, 19 filings with questions among the 21 downloaded (68–549 pages), 40 questions, k=5, two repeats +**Engine:** `cmd/engine --local`, `ingest.mode: toc`, `retrieval.strategy: judgewalk`, abstention off, caches off, Judge on, no generative call anywhere in retrieval +**Question:** the post claims chunk-and-embed cannot be this fast and this exact. Is that measured? + +## Result + +The final run completed all 40 questions and both repeats with no +errors and no engine restart. Every gold evidence page is among the +returned pages for 34 of 40 questions; hit@5 (any returned unit holds +the answer text) is 37 of 40; the answer text is in the first returned +unit for 30 of 40. The six page misses are the four `navbench` had +already found (three Boeing, one Pfizer) plus two where a gold page was +one page off the returned one (Pfizer 70–71 → 71; Verizon 23 and 56 → +23 and 57) — a page-boundary question for the full-page pass, not a +navigation one. + +| system | how it retrieves | F1@5 | hit@5 | answer span in top-1 | p50 / query | $ / query | ingest, 19 filings | deterministic across repeats | +|---|---|---|---|---|---|---|---|---| +| **Vectorless, judgewalk on persisted pages, pages returned** (PR #68, final run 2026-09-21) | Jev ranks the TOC's sections, then page heads, then pages; the evidence pages are returned as-is, ahead of any section | 0.498 | **0.925** | **0.750** | 37 s (p95 75 s) | $0.0040 | 1,111 s (58 s / filing, sub-section splitting on) | 0.38 exact, 0.83 Jaccard | +| Vectorless, judgewalk on persisted pages, sections mapped by page range (PR #68 before HAL-1390) | same navigation; the parser's sections covering the evidence pages returned | 0.197 | 0.475 | 0.000 | 29 s | $0.0040 | — | — | +| Vectorless, judgewalk on the section tree (first pass) | same navigation over the parser's sections and their bodies | 0.453 | 0.650 | 0.575 | 50 s | $0.0067 | 1,080 s (57 s / filing) | 0.43 exact, 0.69 Jaccard | +| chunk-and-embed, BGE-small | 512-token chunks, bge-small-en-v1.5 on the CPU, cosine top-5 | 0.170 | 0.375 | 0.225 | 49 ms | $0 | 1,972 s (104 s / filing, 4 threads) | 1.00 | +| BM25 | 512-token chunks, BM25 top-5 | 0.096 | 0.200 | 0.075 | 48 ms | $0 | 2 s | 1.00 | + +Scoring: a question's gold is FinanceBench's evidence text and answer; a +returned unit is a hit when it contains the gold span (numbers matched +as numbers). F1@5 is the harness's primary quality; "answer span in +top-1" is whether the first returned unit contains it. + +## What the runs taught, in order + +The server's judgewalk was navigating the **parser's section tree** over +section bodies. The Jev-built table of contents was persisted for +treewalk alone, and per-page text was never persisted. That scored +hit@5 0.65 where the same navigation over real pages had scored 0.90 +in `navbench` — the parser's page attribution is exactly what HAL-1375 +had shown to be unreliable. Ingest now persists the pages beside +`documents.toc_tree`, and judgewalk navigates the persisted TOC over +them (PR #68). + +That alone made things **worse**: hit@5 0.475, the answer in the first +result never. The right pages were found and then mapped back to the +parser's sections covering them by page range — the same unreliable +attribution, one step later. Page-based retrieval now returns its +pages (HAL-1390): `/v1/query` leads with the evidence pages as units +of their own, `page` and `confidence` set. On the full 40, hit@5 went +0.475 → 0.925 and the answer in the first unit 0.000 → 0.750. + +Three more things the run surfaced, each fixed on the way: + +- `/v1/query` dropped retrieval's usage, so a client benchmarking + retrieval alone saw $0 (PR #67). +- `ingest.mode: minimal` skips the TOC stage entirely; a `toc` mode now + runs the page-based pipeline and nothing else (PR #67). +- The SDK rejected an abstained response for lacking a model name + (vectorless-sdk #3). Abstention is off for this run so judgewalk's + low-confidence best guess is scored rather than blanked. + +## What the numbers say, and what they do not + +- **Latency.** Chunk-and-embed answers a query in 50 ms; Vectorless on + Jev takes tens of seconds, all of it Judge round-trips. The post's + speed claim is about *ingest* and about the *generative* path it + replaced, not about beating a cosine lookup — say so. +- **Ingest.** Embedding 19 filings with a small model on four CPU + threads took 33 minutes; the Jev TOC stage took 18 minutes for the + same filings with sub-section splitting on, for $0.11. With a GPU the + embedding number collapses; with more Jev concurrency so does ours. +- **Exactness.** Even on the section tree, Vectorless put the answer's + text in its first result 2.6× as often as BGE-small and 7.7× as often + as BM25. That is the citation claim, and it is measured. +- **Determinism.** The baselines are exact across repeats; judgewalk is + not (0.43 exact-match of returned sets), because Jev's probabilities + near the threshold move between calls. A product wants this number + reported, not hidden. +- **Not measured here:** answer quality from the retrieved context + (HAL-1387), the tree-walk chat loop on the same questions, Gemini + embeddings (free-tier quota), and anything on a GPU. + +## Reproduce + +```bash +# engine, local mode, Judge on, page-based pipeline only +VLE_INGEST_MODE=toc TYPESAFE_API_KEY=… ./engine --local -config config.yaml # retrieval.strategy: judgewalk, abstain off, cache off +# bench +cd vectorless-bench && VECTORLESS_BASE_URL=http://localhost:7654 vlbench run --config configs/financebench_jev.yaml +``` diff --git a/docs/evaluations/2026-09-19-leaf-granularity.md b/docs/evaluations/2026-09-19-leaf-granularity.md index cfab008..acf18dc 100644 --- a/docs/evaluations/2026-09-19-leaf-granularity.md +++ b/docs/evaluations/2026-09-19-leaf-granularity.md @@ -62,6 +62,37 @@ Two sources, in order of trust, both "select, don't generate": Operations" to "Contingent Obligations"; its Item 8 became the six statements and twenty-one notes. +## Recursive splitting: a negative result (2026-09-25) + +The splitter ran once over the leaves the contents pass produced, so a +70-page Item 8 split into its notes and a 28-page Note 1 with its own +headings stayed whole. Making it descend through each generation — +same two sources, same per-leaf Judge confirmation, depth-capped — +was built, tested, and measured against the single-pass trees: + +| | one generation | recursive | +|---|---|---| +| leaves per filing, median | 69 | 72 | +| median leaf span | 1 p | 1 p | +| leaf span, p90 | 7 p | 9 p | +| max tree depth | 3 | 4 | +| gold pages inside a leaf | 47 / 47 | 47 / 47 | +| span of the leaf holding a gold page, median | **5 p** | 6 p | +| right section chosen | 40 / 40 | 40 / 40 | +| every gold page in the evidence | **36 / 40** | 34 / 40 | +| pages read / question | 40.6 | 40.8 | +| ingest: Judge requests / cost / wall | **278 / $0.13 / 489 s** | 470 / $0.21 / 1,174 s | + +Nothing improved and ingest paid 1.7× the requests and 2.4× the wall +clock. The reason is visible in the first two rows: after one pass the +median leaf is already a single page, because the per-leaf cap allows +one sub-leaf per two pages. The leaves that remain large are the ones +the splitter could not find headings in — and a second look at the +same text with the same rules does not find them either. + +`SplitGenerations` keeps the capability for a document unlike a 10-K, +defaulting to one. The lesson generalises: recursion is not granularity. + ## What is still missed Four questions, the same four as before the split: Boeing's legal diff --git a/docs/evaluations/2026-09-25-query-latency-and-the-embedding-prefilter.md b/docs/evaluations/2026-09-25-query-latency-and-the-embedding-prefilter.md new file mode 100644 index 0000000..e6d185d --- /dev/null +++ b/docs/evaluations/2026-09-25-query-latency-and-the-embedding-prefilter.md @@ -0,0 +1,123 @@ +# Where the 37 seconds goes, and why a cheap pre-filter cannot take it + +**Date:** 2026-09-25 +**Harness:** direct probes against the provider (`scratchpad/latency_shape.py`), offline recall over cached page text (`page_recall.py`, `within_section.py`), [`cmd/navbench`](../../cmd/navbench/main.go) +**Corpus:** FinanceBench, 19 filings with questions (2,936 pages), 40 questions +**Issues:** HAL-1371, HAL-1542 +**Question:** judgewalk answers in 37 s at the median. Retrieval systems people compare us to answer in 49 ms. How much of the gap is recoverable, and by what? + +## Result: most of it is request shape, not work. A local pre-filter cannot help at any scale; batching can. + +## 1. Request shape barely matters. Two of my own bugs did. + +**First probe, and it was wrong.** Varying pages per request moved two +things at once — the text sent and the number of questions asked — and +the session's first calls were cold. It appeared to show large requests +being punished (17k tokens → 14.3 s). Acting on it, the page-ranking +budget was cut 24k → 6k, and a navigation run came back three times +slower. + +**Second probe, one variable at a time, warm.** Holding state near 6k +while varying question count, then holding questions at 4 while varying +text, both curves are flat: everything lands between 1.4 s and 2.6 s. +The first two calls of a session take 5-6 s and then it settles. Forty +pages as ten parallel requests took 2.6 s wall; the same forty as +sixteen-page requests took 2.6 s each. Shape is not the lever. + +**What was the lever, measured end to end on the same 12 questions:** + +| build | requests / question | median s | hit@5 | +|---|---|---|---| +| baseline (24k budget, tokenised packing) | 4.3 | 37.0 | 12/12 | +| length-estimated packing, len/2 | 6.6 | 35.1 | 12/12 | +| **len/4 packing + limiter starting at 16** | **4.2** | **28.6** | 12/12 | + +Two bugs, both ours: + +- **Packing ran the tokenizer.** Deciding which pages go in which + request tokenised all forty pages first — 4.0 s of CPU before a + single call went out, plus the client tokenising the assembled state + again per request. Packing only needs a safe upper bound, so it now + estimates from length. The first attempt used len/2, which + over-estimates real filing text by two and a half times (measured: + 4.9 characters per token), halved every batch, and sent 6.6 requests + where 4.3 had done — cancelling the saving exactly. len/4 keeps a + fifth of headroom and restores the batch size. +- **The limiter was starting at 4 and being halved by transient + failures.** AIMD needs twenty consecutive successes to widen by one, + which a single interactive query never lives long enough to earn; one + failed request left the run at concurrency 2 for most of its length. + A query's four to seven requests are independent, so navbench now + starts at 16 and the run stayed at 16-18 throughout. + +**What is still unexplained.** At 4.2 requests and ~21k tokens each, +warm rates predict roughly 3 s per request and three sequential phases, +so about 9 s. The engine takes 28.6 s. That factor of three is not +accounted for by anything measured here — candidates are provider +behaviour under sustained load versus a short probe, and retry backoff +after transient failures. It should be instrumented per request before +anyone optimises further, rather than guessed at a third time. + +## 2. A local embedding cannot pre-narrow the document + +The appeal is obvious: if BGE-small could shortlist 30 pages in +milliseconds, Jev would judge 30 instead of reading 40, and two round +trips would collapse into one. Measured offline over all 2,936 pages, +"every gold page inside the top k": + +| k | BGE-small | BM25 | +|---|---|---| +| 5 | 0.500 | 0.075 | +| 10 | 0.675 | 0.150 | +| 20 | 0.775 | 0.225 | +| 30 | 0.800 | 0.250 | +| 50 | 0.875 | 0.300 | + +BGE's ceiling at k=50 is 0.875, below the 0.925 judgewalk already +reaches. A pre-filter that caps the pipeline below where it sits is +not a pre-filter, it is a downgrade. + +**Nor inside the sections the tree picked.** Restricting the ranking to +the sections judgewalk chose (median 120 pages, and they contain every +gold page 40/40): + +| k | any gold page in top k | every gold page in top k | +|---|---|---| +| 10 | 0.750 | 0.700 | +| 20 | 0.825 | 0.775 | +| 40 | 0.950 | **0.875** | + +Picking 40 of those 120 pages by embedding similarity scores 0.875; +Jev reading the heads of the same 120 and picking 40 scores ~0.90+. +The Judge's skim beats the embedding at the same budget. + +So the engine's own two-stage narrowing — a structural prior from the +tree, then a read — is better than semantic similarity at every scale +we can measure. That is the thesis, measured from the other side. + +## 3. What is left of the gap + +Three sequential round trips are inherent to the design: rank sections, +skim, read. Everything else is recoverable: + +- **Our own overhead** (done): tokenised packing and a limiter that + started narrow, together 37 s → 28.6 s on the same questions. +- **A needless chain link** (not yet done): the skim only depends on the + ranking because we skim *the chosen sections*. Skim every page instead + and the two requests fire together — one fewer round trip, and more + pages scored, not fewer. +- **Moving the skim to ingest** (HAL-1542): a per-page index built once + makes the skim a lookup over stored answers. + +None of that reaches 49 ms, and it should not be claimed. A system that +answers in 49 ms does no model work at query time, which is exactly the +compression that costs it 0.375 against our 0.925. The honest target is +single-digit seconds with the accuracy intact. + +## Reproduce + +```bash +python scratchpad/latency_shape.py # needs TYPESAFE_API_KEY; ~30 requests, about a cent +python scratchpad/page_recall.py # offline, cached page text, no API calls +python scratchpad/within_section.py # offline +``` diff --git a/internal/api/abstention_test.go b/internal/api/abstention_test.go index 2018cdf..c21a870 100644 --- a/internal/api/abstention_test.go +++ b/internal/api/abstention_test.go @@ -198,7 +198,7 @@ func TestRespondAbstained(t *testing.T) { confidences := map[tree.SectionID]float64{"sec_a": 0.12, "sec_b": 0.30} rec := httptest.NewRecorder() - d.respondAbstained(rec, tree.DocumentID("doc_x"), "what is x?", confidences, nil) + d.respondAbstained(rec, tree.DocumentID("doc_x"), "what is x?", "", confidences, nil, retrieval.Usage{}) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want 200", rec.Code) @@ -279,7 +279,7 @@ func TestRespondAbstainedTraceTokenAbsent(t *testing.T) { Abstain: config.AbstainBlock{Enabled: true, Below: 0.4}, } rec := httptest.NewRecorder() - d.respondAbstained(rec, tree.DocumentID("doc_x"), "q", map[tree.SectionID]float64{"a": 0.1}, nil) + d.respondAbstained(rec, tree.DocumentID("doc_x"), "q", "", map[tree.SectionID]float64{"a": 0.1}, nil, retrieval.Usage{}) var body map[string]any _ = json.Unmarshal(rec.Body.Bytes(), &body) @@ -338,3 +338,25 @@ func TestRespondAbstainedAnswerSkipsSynthesis(t *testing.T) { var _ = bytes.NewReader var _ = io.EOF var _ = abstentionRouter + +// A page-based strategy's evidence pages are returned as-is, ahead of +// tree sections, with the page number a client can cite (HAL-1390). +func TestEvidencePageSections(t *testing.T) { + t.Parallel() + got := evidencePageSections([]retrieval.EvidencePage{ + {Page: 113, Title: "Note 21 - Legal Proceedings", Text: "A class action filed in 2019 remains pending.", Confidence: 0.91}, + {Page: 7, Text: "Suppliers"}, + }) + if len(got) != 2 { + t.Fatalf("len %d", len(got)) + } + if got[0]["id"] != "page_113" || got[0]["page"] != 113 || got[0]["confidence"] != 0.91 { + t.Errorf("first: %+v", got[0]) + } + if title, _ := got[0]["title"].(string); !strings.Contains(title, "Note 21") || !strings.HasSuffix(title, "p.113") { + t.Errorf("title %q", title) + } + if got[1]["title"] != "Page 7 · p.7" { + t.Errorf("untitled page: %v", got[1]["title"]) + } +} diff --git a/internal/api/server.go b/internal/api/server.go index c619488..a8b6493 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -600,13 +600,26 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { started := time.Now() - plan, _ := d.runPlanner(r.Context(), body.Query, body.EnablePlanning) - ids, confidences, err := d.runSelection(r.Context(), t, plan, body.Query, budget) + // Usage accumulates the way /v1/answer's does: planner, selection, + // re-rank. Span extraction is per section and not counted on either + // endpoint today. + totalUsage := retrieval.Usage{} + plan, planUsage := d.runPlanner(r.Context(), body.Query, body.EnablePlanning) + totalUsage.Add(planUsage) + ids, confidences, selUsage, selResult, err := d.runSelectionResult(r.Context(), t, plan, body.Query, budget) if err != nil { d.Logger.Error("query: strategy failed", "err", err, "document_id", body.DocumentID) writeErr(w, http.StatusInternalServerError, "retrieval failed: "+err.Error()) return } + totalUsage.Add(selUsage) + // The model the caller named, or — when it named none, as a + // Judge-navigated query need not — the strategy, so the field is + // never empty and a client can always tell what answered. + modelUsed := body.Model + if modelUsed == "" { + modelUsed = d.Strategy.Name() + } // Phase 2.4 abstention: if every confident pick is below the // configured threshold, refuse to ground an answer in evidence @@ -615,7 +628,7 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { // responses (no confidences) always fall through to the normal // path so older models keep working. if d.abstentionEnabled(body.EnableAbstain) && shouldAbstain(confidences, d.Abstain.Below) { - d.respondAbstained(w, body.DocumentID, body.Query, confidences, plan) + d.respondAbstained(w, body.DocumentID, body.Query, modelUsed, confidences, plan, totalUsage) return } @@ -647,7 +660,9 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { // never drop sections — at worst the strategy's order is // preserved (see retrieval.ReRanker.ReRank). if d.reRankEnabled(body.EnableReRank) { - enriched, _ = d.runReRank(r.Context(), enriched, body.Query, body.Model) + var reRankUsage retrieval.Usage + enriched, reRankUsage = d.runReRank(r.Context(), enriched, body.Query, body.Model) + totalUsage.Add(reRankUsage) } // Optional: per-section answer-span extraction. Opt-in via config — @@ -660,6 +675,12 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { sections := make([]map[string]any, 0, len(enriched)) finalIDs := make([]tree.SectionID, 0, len(enriched)) + // A page-based strategy's evidence pages lead the list, as-is: the + // page the Judge found holds the answer; a section mapped to it by + // the parser's page attribution may not (HAL-1390). + if selResult != nil { + sections = append(sections, evidencePageSections(selResult.EvidencePages)...) + } for _, e := range enriched { sections = append(sections, sectionWithContentToMap(e)) finalIDs = append(finalIDs, e.sec.ID) @@ -678,14 +699,20 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { "document_id": body.DocumentID, "query": body.Query, "strategy": d.Strategy.Name(), - "model": body.Model, + "model": modelUsed, "sections": sections, "elapsed_ms": time.Since(started).Milliseconds(), "trace_token": traceToken, + // What retrieval cost. /v1/answer always reported this; /v1/query + // dropped it, so a caller benchmarking retrieval alone saw $0. + "usage": usageJSON(totalUsage), } if plan != nil { resp["plan"] = plan } + if selResult != nil && len(selResult.CitedPages) > 0 { + resp["cited_pages"] = selResult.CitedPages + } // Surface the confidence map on the response when present. Only the // finalIDs survive truncation, so trim accordingly. Empty map → // omit so the field stays absent when no signal was available. @@ -1382,24 +1409,33 @@ func (d Deps) runSelectionWithUsage(ctx context.Context, t *tree.Tree, plan *ret // plan is multi-hop AND decomposition is enabled, and surfaces // confidences for the Phase 2.4 abstention check. func (d Deps) runSelectionFull(ctx context.Context, t *tree.Tree, plan *retrieval.Plan, query string, budget retrieval.ContextBudget) ([]tree.SectionID, map[tree.SectionID]float64, retrieval.Usage, error) { + ids, conf, usage, _, err := d.runSelectionResult(ctx, t, plan, query, budget) + return ids, conf, usage, err +} + +// runSelectionResult is runSelectionFull plus the strategy's Result, +// for callers that need what only a page-based strategy carries: +// evidence pages and cited pages. nil when the path had no Result. +func (d Deps) runSelectionResult(ctx context.Context, t *tree.Tree, plan *retrieval.Plan, query string, budget retrieval.ContextBudget) ([]tree.SectionID, map[tree.SectionID]float64, retrieval.Usage, *retrieval.Result, error) { if d.shouldDecompose(plan) { - return retrieval.NewDecomposer(d.Strategy).DecomposedSelectWithConfidences(ctx, t, plan, query, budget) + ids, conf, usage, err := retrieval.NewDecomposer(d.Strategy).DecomposedSelectWithConfidences(ctx, t, plan, query, budget) + return ids, conf, usage, nil, err } if cs, ok := d.Strategy.(retrieval.CostStrategy); ok { res, err := cs.SelectWithCost(ctx, t, query, budget) if err != nil { - return nil, nil, retrieval.Usage{}, err + return nil, nil, retrieval.Usage{}, nil, err } if res == nil { - return nil, nil, retrieval.Usage{}, nil + return nil, nil, retrieval.Usage{}, nil, nil } - return res.SelectedIDs, res.Confidences, res.Usage, nil + return res.SelectedIDs, res.Confidences, res.Usage, res, nil } ids, err := d.Strategy.Select(ctx, t, query, budget) if err != nil { - return nil, nil, retrieval.Usage{}, err + return nil, nil, retrieval.Usage{}, nil, err } - return ids, nil, retrieval.Usage{}, nil + return ids, nil, retrieval.Usage{}, nil, nil } // shouldDecompose returns true when the plan is multi-hop AND @@ -1650,11 +1686,13 @@ const abstentionAnswerText = "I cannot answer this question from the supplied do // the response in the replay log because there's no meaningful // retrieval result to reproduce. Callers replaying an abstention // will simply re-run /v1/query. -func (d Deps) respondAbstained(w http.ResponseWriter, docID tree.DocumentID, query string, confidences map[tree.SectionID]float64, plan *retrieval.Plan) { +func (d Deps) respondAbstained(w http.ResponseWriter, docID tree.DocumentID, query string, model string, confidences map[tree.SectionID]float64, plan *retrieval.Plan, usage retrieval.Usage) { resp := map[string]any{ "document_id": docID, "query": query, "strategy": d.Strategy.Name(), + "model": model, + "usage": usageJSON(usage), "sections": []any{}, "abstained": true, "abstention_reason": abstentionReason, @@ -1667,6 +1705,40 @@ func (d Deps) respondAbstained(w http.ResponseWriter, docID tree.DocumentID, que writeJSON(w, http.StatusOK, resp) } +// evidencePageSections renders a page-based strategy's evidence pages +// in the sections shape: id "page_", the owning section's title with +// the page number, the page's text, and the Judge's confidence. +func evidencePageSections(pages []retrieval.EvidencePage) []map[string]any { + out := make([]map[string]any, 0, len(pages)) + for _, ep := range pages { + title := ep.Title + if title == "" { + title = fmt.Sprintf("Page %d", ep.Page) + } + out = append(out, map[string]any{ + "id": fmt.Sprintf("page_%d", ep.Page), + "title": fmt.Sprintf("%s · p.%d", title, ep.Page), + "content": ep.Text, + "page": ep.Page, + "confidence": ep.Confidence, + "token_count": len(ep.Text) / 4, + }) + } + return out +} + +// usageJSON is the wire shape of retrieval.Usage on /v1/query, the same +// keys /v1/answer uses for total_usage. +func usageJSON(u retrieval.Usage) map[string]any { + return map[string]any{ + "input_tokens": u.InputTokens, + "output_tokens": u.OutputTokens, + "total_tokens": u.TotalTokens, + "cost_usd": u.CostUSD, + "llm_calls": u.LLMCalls, + } +} + // respondAbstainedAnswer writes the abstention shape for /v1/answer. // The answer text is the canonical refusal; citations is empty; // usage carries the LLM tokens spent up to the abstention point diff --git a/pkg/config/config.go b/pkg/config/config.go index b334469..d9b5cdd 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1319,7 +1319,7 @@ func (c Config) Validate() error { } switch c.Retrieval.Strategy { - case "auto", "single-pass", "chunked-tree", "agentic", "treewalk": + case "auto", "single-pass", "chunked-tree", "agentic", "treewalk", "judgewalk": default: return fmt.Errorf("unknown retrieval.strategy: %q", c.Retrieval.Strategy) } @@ -1339,9 +1339,9 @@ func (c Config) Validate() error { } switch c.Ingest.Mode { - case "", "full", "minimal": + case "", "full", "minimal", "toc": default: - return fmt.Errorf("ingest.mode must be one of full|minimal, got %q", c.Ingest.Mode) + return fmt.Errorf("ingest.mode must be one of full|minimal|toc, got %q", c.Ingest.Mode) } if c.Ingest.HyDE.NumQuestions < 0 { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 9a3bb34..7761240 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -652,7 +652,7 @@ func TestValidateLLMDrivers(t *testing.T) { func TestValidateRetrievalStrategy(t *testing.T) { t.Parallel() - for _, s := range []string{"auto", "single-pass", "chunked-tree", "agentic", "treewalk"} { + for _, s := range []string{"auto", "single-pass", "chunked-tree", "agentic", "treewalk", "judgewalk"} { cfg := Default() cfg.Database.URL = "postgres://localhost/test" cfg.Retrieval.Strategy = s diff --git a/pkg/ingest/ingest.go b/pkg/ingest/ingest.go index 4d2b6e0..41a4869 100644 --- a/pkg/ingest/ingest.go +++ b/pkg/ingest/ingest.go @@ -50,6 +50,14 @@ import ( // and table extraction. Any other value runs the full pipeline. const ModeMinimal = "minimal" +// ModeTOC is parse → build tree → table of contents → persist → ready: +// the page-based pipeline and nothing else. On a Judge the TOC stage is +// three requests and seconds; the per-section generative enrichment +// (summarize, HyDE, multi-axis) that full mode adds is minutes and is +// not used by page-based retrieval. Table extraction is skipped as in +// minimal mode: the page text still carries the tables' text. +const ModeTOC = "toc" + // docPersister is the narrow slice of *db.Pool the parse → persist → // ready path depends on. Declaring it here (rather than threading the // concrete *db.Pool) lets the minimal-mode runner be exercised with a @@ -377,7 +385,7 @@ func (p *Pipeline) Handler() queue.Handler { // tree → persist → ready, with no LLM enrichment and no table // extraction. Otherwise it runs the full enrichment pipeline below. func (p *Pipeline) Run(ctx context.Context, pl Payload) error { - if p.Mode == ModeMinimal { + if p.Mode == ModeMinimal || p.Mode == ModeTOC { return p.runMinimal(ctx, p.DB, pl) } @@ -462,6 +470,14 @@ func (p *Pipeline) runTOCBuilder(ctx context.Context, docID tree.DocumentID, par log.Info("ingest: toc-builder skipped; no per-page text available") return nil } + // The pages are the ground truth every page-level stage reasons + // over (HAL-1375). Persist them beside the table of contents so + // retrieval navigates the same text ingest did, not a per-section + // reconstruction of it. Non-fatal: without them judgewalk falls + // back to the section tree. + if err := p.persistPages(ctx, docID, pages); err != nil { + log.Warn("ingest: pages not persisted; judgewalk will use the section tree", "err", err) + } model := p.TOCModel if model == "" { model = p.SummaryModel @@ -730,13 +746,20 @@ func (p *Pipeline) runMinimal(ctx context.Context, store docPersister, pl Payloa return err } - // Skip summarize / HyDE / multi-axis / TOC entirely — flip straight - // to ready. The document is now queryable via the page-based - // strategy (synthesised TOC + raw page reads). + // Minimal mode skips summarize / HyDE / multi-axis / TOC entirely + // and flips straight to ready; the document is queryable via the + // page-based strategy on a TOC synthesised from the section tree. + // TOC mode builds the real table of contents first — same builder + // and persistence as full mode, non-fatal for the same reason. + if p.Mode == ModeTOC && pl.ContentType == "application/pdf" { + if err := p.runTOCBuilder(ctx, pl.DocumentID, parsed, log); err != nil { + log.Warn("ingest: toc-builder failed; falling back to NULL toc_tree", "err", err) + } + } if err := store.SetDocumentStatus(ctx, pl.DocumentID, db.StatusReady, ""); err != nil { return err } - log.Info("ingest: ready (minimal mode)") + log.Info("ingest: ready (" + p.Mode + " mode)") return nil } @@ -1282,6 +1305,23 @@ func NewDocumentID() tree.DocumentID { // SourceKey returns the canonical storage key where an ingest payload's // original bytes live. +// PagesKey is where a document's per-page text lives in storage: a +// JSON array of {page_number, text}, written by ingest for paged +// documents and read by page-based retrieval. +func PagesKey(id tree.DocumentID) string { return "pages/" + string(id) + ".json" } + +// persistPages writes the per-page text to storage at PagesKey. +func (p *Pipeline) persistPages(ctx context.Context, docID tree.DocumentID, pages []PageText) error { + if p.Storage == nil { + return fmt.Errorf("no storage") + } + raw, err := json.Marshal(pages) + if err != nil { + return err + } + return p.Storage.Put(ctx, PagesKey(docID), bytes.NewReader(raw), storage.Metadata{ContentType: "application/json", Size: int64(len(raw))}) +} + func SourceKey(id tree.DocumentID, filename string) string { // Keep the original extension so future content-type sniffing works. ext := path.Ext(filename) diff --git a/pkg/ingest/minimal_mode_test.go b/pkg/ingest/minimal_mode_test.go index e753b1b..6e4f1e9 100644 --- a/pkg/ingest/minimal_mode_test.go +++ b/pkg/ingest/minimal_mode_test.go @@ -3,6 +3,7 @@ package ingest import ( "bytes" "context" + "encoding/json" "io" "log/slog" "os" @@ -315,3 +316,65 @@ func reconstructTree(_ tree.DocumentID, title string, rows []db.Section) *tree.S return &tree.Section{Title: title, Children: topLevel} } } + +// TOC mode on a non-PDF is minimal mode: nothing for the TOC builder to +// do, no model call, ready. +func TestTOCModeOnMarkdownMakesNoLLMCall(t *testing.T) { + ctx := context.Background() + st, err := storage.NewLocal(t.TempDir()) + if err != nil { + t.Fatal(err) + } + body := []byte("# Title\n\n## Section A\n\nAlpha.\n\n## Section B\n\nBeta.\n") + docID := NewDocumentID() + srcKey := SourceKey(docID, "doc.md") + if err := st.Put(ctx, srcKey, bytes.NewReader(body), storage.Metadata{ContentType: "text/markdown", Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + p := NewPipeline(Pipeline{ + Storage: st, + LLM: &failIfCalledLLM{t: t}, + Parsers: DefaultRegistry(), + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + Mode: ModeTOC, + TOCEnabled: true, + }) + fake := &fakeDocStore{} + if err := p.runMinimal(ctx, fake, Payload{DocumentID: docID, ContentType: "text/markdown", Filename: "doc.md", SourceRef: srcKey}); err != nil { + t.Fatalf("run: %v", err) + } + status, errMsg, sections := fake.snapshot() + if status != db.StatusReady { + t.Fatalf("status %q (%q) want ready", status, errMsg) + } + if len(sections) == 0 { + t.Fatal("no sections persisted") + } +} + +// Pages are persisted beside the table of contents as JSON at PagesKey, +// in the shape page-based retrieval reads back. +func TestPersistPagesRoundTrip(t *testing.T) { + ctx := context.Background() + st, err := storage.NewLocal(t.TempDir()) + if err != nil { + t.Fatal(err) + } + p := &Pipeline{Storage: st} + pages := []PageText{{PageNumber: 1, Text: "cover"}, {PageNumber: 3, Text: "Item 1. Business"}} + if err := p.persistPages(ctx, "doc_x", pages); err != nil { + t.Fatal(err) + } + rc, _, err := st.Get(ctx, PagesKey("doc_x")) + if err != nil { + t.Fatal(err) + } + defer rc.Close() + var back []PageText + if err := json.NewDecoder(rc).Decode(&back); err != nil { + t.Fatal(err) + } + if len(back) != 2 || back[1].PageNumber != 3 || back[1].Text != "Item 1. Business" { + t.Errorf("round trip: %+v", back) + } +} diff --git a/pkg/ingest/toc_builder.go b/pkg/ingest/toc_builder.go index 2f404e6..73d716c 100644 --- a/pkg/ingest/toc_builder.go +++ b/pkg/ingest/toc_builder.go @@ -23,8 +23,8 @@ import ( // pages produced by the existing parser pipeline and over synthetic // fixtures used in tests. type PageText struct { - PageNumber int - Text string + PageNumber int `json:"page_number"` + Text string `json:"text"` } // TOCBuilder builds an LLM-derived table-of-contents tree for a @@ -111,6 +111,12 @@ type TOCBuilder struct { // negative disables splitting. SplitLeavesOver int + // SplitGenerations is how many times a path may be split: 1 (the + // default) splits the leaves the contents pass produced and stops. + // More descends into the sub-leaves it creates, which measured as + // pure cost on FinanceBench — see defaultSplitGenerations. + SplitGenerations int + // DetectChars caps the characters of each page sent to detection when // MinimalContext is on. Zero means detectCharsMinimal. A contents page // declares itself in its first couple of thousand characters; the diff --git a/pkg/ingest/toc_split.go b/pkg/ingest/toc_split.go index 6615215..6f45df5 100644 --- a/pkg/ingest/toc_split.go +++ b/pkg/ingest/toc_split.go @@ -57,10 +57,35 @@ var ( reSplitNumbered = regexp.MustCompile(`(?i)^(note|item|section|part)\s+\d+[a-c]?\b`) ) +// defaultSplitGenerations is how many times the splitter may split +// along one path. One means a single pass: split the leaves the +// contents pass produced, and stop. +// +// Measured on FinanceBench (2026-09-25): descending into the sub-leaves +// bought nothing and cost plenty. Leaves per filing 69 → 72, median +// span 1 page either way, the leaf holding a gold page 5 → 6 pages, +// coverage 47/47 both, navigation's right-section rate 40/40 both and +// its evidence rate 36/40 → 34/40, with pages read per question +// unchanged at ~41. Ingest paid 278 → 470 Judge requests, $0.13 → +// $0.21, and 489 → 1,174 seconds of wall clock. A second generation +// only reaches leaves the first pass could not find headings in, and +// those are exactly the ones a second look does not help. +// +// The capability stays because a document unlike a 10-K — a deep +// standard, a long contract — may need it; it is off until a corpus +// shows it earning its cost. +const defaultSplitGenerations = 1 + +// splitMaxDepth caps the tree depth the splitter may reach regardless +// of generations, so a pathological document cannot recurse into +// paragraphs. +const splitMaxDepth = 6 + // splitLargeLeaves walks the tree and splits every leaf whose span -// exceeds over pages. It runs after end pages are derived, so spans are -// known, and re-derives them for the children it adds. Returns how many -// sub-leaves were added. +// exceeds over pages, then splits the sub-leaves it made, until no +// leaf is over the threshold or splitMaxDepth is reached. Runs after +// end pages are derived, so spans are known, and re-derives them for +// each generation of children. Returns how many sub-leaves were added. func (b *TOCBuilder) splitLargeLeaves(ctx context.Context, nodes []tree.TOCNode, pages []PageText, over int, usage *Usage) int { if over <= 0 || b.Judge == nil { return 0 @@ -69,13 +94,17 @@ func (b *TOCBuilder) splitLargeLeaves(ctx context.Context, nodes []tree.TOCNode, for _, p := range pages { byPage[p.PageNumber] = p.Text } + maxGen := b.splitGenerations() added := 0 - var walk func(ns []tree.TOCNode) - walk = func(ns []tree.TOCNode) { + var walk func(ns []tree.TOCNode, depth, gen int) + walk = func(ns []tree.TOCNode, depth, gen int) { for i := range ns { n := &ns[i] if len(n.Nodes) > 0 { - walk(n.Nodes) + walk(n.Nodes, depth+1, gen) + continue + } + if depth >= splitMaxDepth || gen >= maxGen { continue } if n.StartPage <= 0 || n.EndPage < n.StartPage || n.EndPage-n.StartPage+1 <= over { @@ -103,12 +132,26 @@ func (b *TOCBuilder) splitLargeLeaves(ctx context.Context, nodes []tree.TOCNode, deriveEndPagesIn(subs, n.EndPage) n.Nodes = subs added += len(subs) + // Descend into what was just made, when more generations are + // allowed: a 70-page Item 8 splits into notes, and a 28-page + // note has its own headings. + walk(n.Nodes, depth+1, gen+1) } } - walk(nodes) + walk(nodes, 1, 0) return added } +// splitGenerations resolves SplitGenerations: zero selects the default +// of one pass, negative is treated as one, and any larger value allows +// that many generations of splitting along a path. +func (b *TOCBuilder) splitGenerations() int { + if b.SplitGenerations > 0 { + return b.SplitGenerations + } + return defaultSplitGenerations +} + // splitLeaf returns the sub-leaves of one leaf, with start pages, in // page order. func (b *TOCBuilder) splitLeaf(ctx context.Context, leaf *tree.TOCNode, pages []PageText, byPage map[int]string, over int, usage *Usage) ([]tree.TOCNode, error) { diff --git a/pkg/ingest/toc_split_test.go b/pkg/ingest/toc_split_test.go index b041e37..bc37a75 100644 --- a/pkg/ingest/toc_split_test.go +++ b/pkg/ingest/toc_split_test.go @@ -2,6 +2,7 @@ package ingest import ( "context" + "fmt" "strings" "testing" @@ -255,3 +256,117 @@ func titlesOfNodes(ns []tree.TOCNode) []string { } return out } + +// A 70-page Item 8 splits into notes, and a note long enough to have +// its own headings splits again — the same two sources, one level down. +func TestSplitRecursesIntoItsOwnSubLeaves(t *testing.T) { + ps := []PageText{ + {54, "Item 8. Financial Statements\nIndex to the Consolidated Financial Statements\nPage\nNote 1 - Summary of Significant Accounting Policies 60\nNote 2 - Goodwill 95"}, + } + for p := 55; p <= 100; p++ { + text := "Table of Contents\nprose about accounting " + strings.Repeat("x ", 40) + switch p { + case 60: + text = "Note 1 - Summary of Significant Accounting Policies\nPrinciples of Consolidation\nThe consolidated statements include." + case 72: + text = "Table of Contents\nRevenue and Related Cost Recognition\nWe recognize revenue when control transfers." + case 84: + text = "Table of Contents\nUse of Estimates\nEstimates are used throughout." + case 95: + text = "Note 2 - Goodwill\nGoodwill is tested annually." + } + ps = append(ps, PageText{p, text}) + } + b := &TOCBuilder{Judge: splitJudge("note", "principles of consolidation", "revenue and related", "use of estimates"), SplitGenerations: 3} + nodes := []tree.TOCNode{{Structure: "2.5", Title: "Item 8. Financial Statements", StartPage: 54, EndPage: 100}} + var usage Usage + if n := b.splitLargeLeaves(context.Background(), nodes, ps, 20, &usage); n < 4 { + t.Fatalf("sub-leaves added: %d", n) + } + item8 := nodes[0] + if len(item8.Nodes) < 2 { + t.Fatalf("Item 8 children: %v", titlesOfNodes(item8.Nodes)) + } + var note1 *tree.TOCNode + for i := range item8.Nodes { + if strings.HasPrefix(item8.Nodes[i].Title, "Note 1") { + note1 = &item8.Nodes[i] + } + } + if note1 == nil { + t.Fatalf("Note 1 missing: %v", titlesOfNodes(item8.Nodes)) + } + // Note 1 spans 60–94, over the threshold, so it split again. + if len(note1.Nodes) < 2 { + t.Errorf("Note 1 (%d-%d) should have split at its own headings, children: %v", + note1.StartPage, note1.EndPage, titlesOfNodes(note1.Nodes)) + } + for _, c := range note1.Nodes { + if !strings.HasPrefix(c.Structure, note1.Structure+".") { + t.Errorf("grandchild structure %q does not nest under %q", c.Structure, note1.Structure) + } + if c.StartPage < note1.StartPage || c.EndPage > note1.EndPage { + t.Errorf("grandchild %q spans %d-%d, outside its parent %d-%d", c.Title, c.StartPage, c.EndPage, note1.StartPage, note1.EndPage) + } + } +} + +// The depth cap stops the descent even when leaves stay oversized. +func TestSplitStopsAtMaxDepth(t *testing.T) { + var ps []PageText + for p := 1; p <= 120; p++ { + text := "Section Heading " + fmt.Sprint(p%7) + "\nprose " + strings.Repeat("y ", 40) + ps = append(ps, PageText{p, text}) + } + b := &TOCBuilder{Judge: splitJudge("section heading"), SplitGenerations: 99} + nodes := []tree.TOCNode{{Structure: "1", Title: "Everything", StartPage: 1, EndPage: 120}} + var usage Usage + b.splitLargeLeaves(context.Background(), nodes, ps, 20, &usage) + var deepest func(ns []tree.TOCNode, d int) int + deepest = func(ns []tree.TOCNode, d int) int { + max := d + for _, n := range ns { + if len(n.Nodes) > 0 { + if x := deepest(n.Nodes, d+1); x > max { + max = x + } + } + } + return max + } + if got := deepest(nodes, 1); got > splitMaxDepth { + t.Errorf("tree reached depth %d, cap is %d", got, splitMaxDepth) + } +} + +// One generation is the default: the leaves the contents pass produced +// are split, and the sub-leaves are left alone however large they are. +func TestSplitIsOneGenerationByDefault(t *testing.T) { + ps := []PageText{ + {54, "Item 8. Financial Statements\nIndex to the Consolidated Financial Statements\nPage\nNote 1 - Summary of Significant Accounting Policies 60\nNote 2 - Goodwill 95"}, + } + for p := 55; p <= 100; p++ { + text := "Table of Contents\nprose " + strings.Repeat("x ", 40) + switch p { + case 60: + text = "Note 1 - Summary of Significant Accounting Policies\nPrinciples of Consolidation\nThe statements include." + case 72: + text = "Table of Contents\nRevenue and Related Cost Recognition\nWe recognize revenue when control transfers." + case 95: + text = "Note 2 - Goodwill\nGoodwill is tested annually." + } + ps = append(ps, PageText{p, text}) + } + b := &TOCBuilder{Judge: splitJudge("note", "principles of consolidation", "revenue and related")} + nodes := []tree.TOCNode{{Structure: "2.5", Title: "Item 8. Financial Statements", StartPage: 54, EndPage: 100}} + var usage Usage + b.splitLargeLeaves(context.Background(), nodes, ps, 20, &usage) + for _, c := range nodes[0].Nodes { + if len(c.Nodes) > 0 { + t.Errorf("%q split a second time under the default of one generation: %v", c.Title, titlesOfNodes(c.Nodes)) + } + } + if len(nodes[0].Nodes) < 2 { + t.Errorf("the first generation should still split: %v", titlesOfNodes(nodes[0].Nodes)) + } +} diff --git a/pkg/retrieval/judgewalk.go b/pkg/retrieval/judgewalk.go index a8be4f8..e035ef0 100644 --- a/pkg/retrieval/judgewalk.go +++ b/pkg/retrieval/judgewalk.go @@ -2,6 +2,7 @@ package retrieval import ( "context" + "encoding/json" "fmt" "sort" "strings" @@ -10,7 +11,6 @@ import ( "regexp" "github.com/hallelx2/llmgate" - "github.com/hallelx2/llmgate/judge/typesafe" "github.com/hallelx2/vectorless-engine/pkg/tree" ) @@ -111,17 +111,34 @@ type JudgeNavigator struct { // The provider treats the whole state object as shared context for // every question — there is no per-question state — so state plus // the longest question must stay under 32k tokens. Zero selects - // 24k, the same ceiling the TOC stage uses; that is six to eight - // dense filing pages per request. + // defaultNavReqTokens. + // + // This is a LATENCY knob, not just a limit. Batches are sent + // concurrently, so a smaller budget means more requests in flight, + // not more waiting — and the provider is far happier with many small + // requests than a few large ones (see defaultNavReqTokens). RequestBudgetTokens int } const ( - defaultNavThreshold = 0.5 - defaultNavMaxPages = 40 - defaultNavCoarse = 120 - defaultNavHeadChars = 700 - defaultNavPageChars = 6000 + defaultNavThreshold = 0.5 + defaultNavMaxPages = 40 + defaultNavCoarse = 120 + defaultNavHeadChars = 700 + defaultNavPageChars = 6000 + // defaultNavReqTokens bounds one request's state. Measured warm on + // 2026-09-25, one Noul per page: request SHAPE barely matters. + // Forty pages went out as ten parallel requests in 2.6 s wall, and + // as sixteen-page requests in 2.6 s each. An earlier probe appeared + // to punish large requests; that was cold-start contamination — the + // first calls of a session take 5-6 s and the rest settle to + // 1.5-2.6 s. + // + // This was briefly retuned to 6k on the strength of that bad probe + // and one navigation run came back three times slower (not a paired + // control either). No measured reason to move off 24k, which stays + // under the provider's 32k per-question ceiling with room for the + // question text. defaultNavReqTokens = 24_000 navLeafBatch = 120 navMinEvidencePages = 2 @@ -497,14 +514,26 @@ func (n *JudgeNavigator) Navigate(ctx context.Context, query string, leaves []Na return out, nil } -// countTokens uses the provider's tokenizer, so the batch budget is -// measured the way the request will be. Dense financial tables run -// near one token per two characters; a bytes/4 guess overflowed. +// countTokens estimates a string's token count for packing batches. +// +// It deliberately does NOT run the provider's tokenizer. Measured +// 2026-09-25: tokenising one question's forty pages costs 4.0 s of +// CPU, and the client tokenises the assembled state again before every +// request — about five seconds per question spent counting rather than +// asking, on a query whose median is thirty-seven. +// +// The packing budget only needs a safe upper bound; the client's own +// check is exact and refuses anything over the ceiling. +// +// Measured on real filing pages: 24,000 characters of page text bill +// as 4,875 tokens, so 4.9 characters per token. len/2 was the first +// guess and over-estimated by two and a half times, halving every +// batch and sending 6.6 requests per question where 4.3 had done — +// which cancelled the CPU saved. len/4 leaves a fifth of headroom +// over the measured ratio, which covers dense numeric tables without +// throwing away batch size. func countTokens(text string) int { - if n, err := typesafe.EstimateTokens(text); err == nil { - return n - } - return len(text)/3 + 1 + return len(text)/4 + 1 } // reReference finds the cross-references a page makes: "see Note 21", @@ -561,6 +590,22 @@ func judgeUsage(res *llmgate.Judgment) Usage { type JudgeWalkStrategy struct { Navigator JudgeNavigator PageLoader PageContentLoader + + // TOC and Pages, when both are set and both have data for the + // document, are what navigation runs over: the table of contents + // ingest built (leaves with real page ranges, sub-sections and all) + // and the per-page text ingest persisted beside it. That is the + // pipeline the FinanceBench evaluations measured. Without either, + // navigation falls back to the section tree and section bodies — + // the parser's page attribution, which is what HAL-1375 showed to + // be unreliable — and scored 0.65 where the pages scored 0.90. + TOC TOCProvider + Pages PageStore +} + +// PageStore serves a document's per-page text, as ingest persisted it. +type PageStore interface { + LoadPages(ctx context.Context, docID tree.DocumentID) ([]NavPage, error) } const strategyNameJudgeWalk = "judgewalk" @@ -582,10 +627,121 @@ func (s *JudgeWalkStrategy) Select(ctx context.Context, t *tree.Tree, query stri } // SelectWithCost runs navigation and reports usage. -func (s *JudgeWalkStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, query string, _ ContextBudget) (*Result, error) { +func (s *JudgeWalkStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, query string, budget ContextBudget) (*Result, error) { if t == nil || t.Root == nil { return &Result{}, nil } + if res, ok := s.selectOnPersistedPages(ctx, t, query); ok { + return res, nil + } + return s.selectOnSectionTree(ctx, t, query, budget) +} + +// selectOnPersistedPages navigates the persisted table of contents over +// the persisted pages. ok is false when either is missing, so the +// caller falls back; an error from navigation itself is returned as a +// Result error through ok=true. +func (s *JudgeWalkStrategy) selectOnPersistedPages(ctx context.Context, t *tree.Tree, query string) (*Result, bool) { + if s.TOC == nil || s.Pages == nil { + return nil, false + } + raw, err := s.TOC.GetTOC(ctx, t.DocumentID) + if err != nil || len(raw) == 0 { + return nil, false + } + var nodes []tree.TOCNode + if err := json.Unmarshal(raw, &nodes); err != nil || len(nodes) == 0 { + return nil, false + } + pages, err := s.Pages.LoadPages(ctx, t.DocumentID) + if err != nil || len(pages) == 0 { + return nil, false + } + byNum := make(map[int]NavPage, len(pages)) + for _, p := range pages { + byNum[p.Number] = p + } + var leaves []NavLeaf + var walk func(ns []tree.TOCNode, path string) + walk = func(ns []tree.TOCNode, path string) { + for _, n := range ns { + p := n.Title + if path != "" { + p = path + " > " + n.Title + } + if len(n.Nodes) > 0 { + walk(n.Nodes, p) + continue + } + if n.StartPage > 0 && n.EndPage >= n.StartPage { + leaves = append(leaves, NavLeaf{ID: n.NodeID, Title: n.Title, Path: p, Start: n.StartPage, End: n.EndPage, Summary: n.Summary}) + } + } + } + walk(nodes, "") + if len(leaves) == 0 { + return nil, false + } + load := func(_ context.Context, leaf NavLeaf) ([]NavPage, error) { + var ps []NavPage + for n := leaf.Start; n <= leaf.End; n++ { + if p, ok := byNum[n]; ok { + ps = append(ps, p) + } + } + return ps, nil + } + nav, err := s.Navigator.Navigate(ctx, query, leaves, load) + if err != nil { + return &Result{ModelUsed: "judge"}, false + } + // The API answers in sections. Each evidence page names the sections + // that cover it; the cited pages are the evidence pages themselves. + sections := flattenSectionsByPage(t) + var ranges []pageRange + conf := map[tree.SectionID]float64{} + var ids []tree.SectionID + seen := map[tree.SectionID]bool{} + for _, ev := range nav.Evidence { + pg := ev.Page.Number + ranges = append(ranges, pageRange{Start: pg, End: pg}) + for _, id := range sectionsOverlapping(sections, []pageRange{{Start: pg, End: pg}}) { + if !seen[id] { + seen[id] = true + ids = append(ids, id) + } + if ev.P > conf[id] { + conf[id] = ev.P + } + } + } + best := 0.0 + if len(nav.Evidence) > 0 { + best = nav.Evidence[0].P + } + leafTitle := map[string]string{} + for _, l := range leaves { + leafTitle[l.ID] = l.Title + } + evidence := make([]EvidencePage, 0, len(nav.Evidence)) + for _, ev := range nav.Evidence { + evidence = append(evidence, EvidencePage{Page: ev.Page.Number, Title: leafTitle[ev.Page.LeafID], Text: ev.Page.Text, Confidence: ev.P}) + } + return &Result{ + SelectedIDs: ids, + Confidences: conf, + Confidence: best, + CitedPages: rangesToPairs(ranges), + EvidencePages: evidence, + ModelUsed: "judge", + Usage: nav.Usage, + HopsTaken: nav.Requests, + }, true +} + +// selectOnSectionTree is the fallback: the parser's section tree, each +// section's body chunked into page-sized units. +func (s *JudgeWalkStrategy) selectOnSectionTree(ctx context.Context, t *tree.Tree, query string, _ ContextBudget) (*Result, error) { sections := flattenSectionsByPage(t) byID := map[string]sectionPageEntry{} paths := sectionPaths(t) diff --git a/pkg/retrieval/judgewalk_test.go b/pkg/retrieval/judgewalk_test.go index 12552ce..0ef577e 100644 --- a/pkg/retrieval/judgewalk_test.go +++ b/pkg/retrieval/judgewalk_test.go @@ -309,3 +309,61 @@ func TestNavigateFillsThePageBudgetOnAFineTree(t *testing.T) { t.Errorf("full read should be MaxPages=10, got %d", len(res.Pages)) } } + +type fakeTOC struct{ raw []byte } + +func (f fakeTOC) GetTOC(context.Context, tree.DocumentID) ([]byte, error) { return f.raw, nil } + +type fakePages struct{ pages []NavPage } + +func (f fakePages) LoadPages(context.Context, tree.DocumentID) ([]NavPage, error) { + return f.pages, nil +} + +// With a persisted table of contents and pages, navigation runs over +// them — sub-sections, real page ranges — and the API still gets the +// sections that cover the evidence pages, plus the pages themselves. +func TestJudgeWalkUsesThePersistedTOCAndPages(t *testing.T) { + j, _ := navJudge("note 21", "class action") + s := NewJudgeWalkStrategy(j) + s.TOC = fakeTOC{raw: []byte(`[{"node_id":"toc_2","title":"PART II","start_page":21,"end_page":126,"nodes":[ + {"node_id":"toc_2_5","title":"Item 8. Financial Statements","start_page":54,"end_page":125,"nodes":[ + {"node_id":"toc_2_5_1","title":"Note 1 - Policies","start_page":63,"end_page":73}, + {"node_id":"toc_2_5_21","title":"Note 21 - Legal Proceedings","start_page":113,"end_page":114}]}, + {"node_id":"toc_2_6","title":"Item 9. Changes","start_page":126,"end_page":126}]}]`)} + var pages []NavPage + for p := 21; p <= 126; p++ { + text := "prose" + if p == 113 { + text = "Note 21 - Legal Proceedings\nA class action filed in 2019 remains pending." + } + pages = append(pages, NavPage{Number: p, Text: text}) + } + s.Pages = fakePages{pages: pages} + // The section tree the API answers in: Item 8's body is one section + // spanning 54–125 as the parser saw it. + tr := &tree.Tree{DocumentID: "d", Root: &tree.Section{ID: "root", Children: []*tree.Section{ + {ID: "s7", Title: "Item 7", PageStart: 22, PageEnd: 53, ContentRef: "r7"}, + {ID: "s8", Title: "Item 8", PageStart: 54, PageEnd: 125, ContentRef: "r8"}, + }}} + res, err := s.SelectWithCost(context.Background(), tr, "legal proceedings?", ContextBudget{}) + if err != nil { + t.Fatal(err) + } + if len(res.SelectedIDs) != 1 || res.SelectedIDs[0] != "s8" { + t.Errorf("sections covering the evidence page: %v want [s8]", res.SelectedIDs) + } + if len(res.CitedPages) == 0 || res.CitedPages[0] != [2]int{113, 113} { + t.Errorf("cited pages should be the evidence pages: %v", res.CitedPages) + } + if res.ModelUsed != "judge" || res.Usage.LLMCalls == 0 { + t.Errorf("usage/model: %+v", res) + } + // Without a page store it falls back to the section tree and still answers. + s.Pages = nil + s.PageLoader = mapLoader{"r8": "Note 21 - Legal Proceedings\nA class action filed in 2019 remains pending.", "r7": "prose"} + res, err = s.SelectWithCost(context.Background(), tr, "legal proceedings?", ContextBudget{}) + if err != nil || len(res.SelectedIDs) == 0 { + t.Errorf("fallback path: %v %+v", err, res) + } +} diff --git a/pkg/retrieval/strategy.go b/pkg/retrieval/strategy.go index 2ddd526..6647a91 100644 --- a/pkg/retrieval/strategy.go +++ b/pkg/retrieval/strategy.go @@ -98,6 +98,13 @@ type Result struct { // 102-104 leaves a concrete page footprint behind. PagesRead []PageReadEntry `json:"pages_read,omitempty"` + // EvidencePages are the pages a page-based strategy judged to hold + // the answer, with their text — the unit such a strategy actually + // found, returned as-is rather than mapped back to sections whose + // page attribution may be wrong (HAL-1390). Empty for section-based + // strategies. + EvidencePages []EvidencePage `json:"evidence_pages,omitempty"` + // CitedPages is the FINAL set of page ranges the answer commits // to — the model's cited_pages after dedup and the confidence cap, // NOT every page it read (that is PagesRead). Page-based strategies @@ -117,6 +124,14 @@ type Result struct { Confidence float64 `json:"confidence,omitempty"` } +// EvidencePage is one page a page-based strategy returns as evidence. +type EvidencePage struct { + Page int `json:"page"` + Title string `json:"title,omitempty"` // the section the page belongs to + Text string `json:"text"` + Confidence float64 `json:"confidence"` +} + // PageReadEntry is one get_pages tool call that materialised during a // page-based retrieval loop. StartPage and EndPage are inclusive, // 1-indexed. SectionIDs lists every section whose [PageStart,PageEnd]