judgewalk navigates the persisted TOC over persisted pages, and returns pages (HAL-1371, HAL-1390)

cmd/engine/main.go

@@ -7,6 +7,7 @@ package main
 import (
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -36,6 +37,7 @@ import (
 	"github.com/hallelx2/vectorless-engine/pkg/queue"
 	"github.com/hallelx2/vectorless-engine/pkg/retrieval"
 	"github.com/hallelx2/vectorless-engine/pkg/storage"
+	"github.com/hallelx2/vectorless-engine/pkg/tree"
 )
 
 // version is set at build time via -ldflags "-X main.version=..."
@@ -141,7 +143,7 @@ func run() error {
 	} else {
 		logger.Warn("judge: none configured — TOC judgements run on the generative driver, one call per page (set TYPESAFE_API_KEY)")
 	}
-	strategy := buildStrategy(cfg.Retrieval, llmClient, judge, store)
+	strategy := buildStrategy(cfg.Retrieval, llmClient, judge, store, pool)
 
 	// Wrap with caching if enabled.
 	if cfg.Retrieval.Cache.Enabled {
@@ -249,6 +251,8 @@ func run() error {
 	})
 	if cfg.Ingest.Mode == ingest.ModeMinimal {
 		logger.Info("ingest: MINIMAL mode — parse→persist→ready; skipping summarize/HyDE/multi-axis/TOC + table extraction")
+	} else if cfg.Ingest.Mode == ingest.ModeTOC {
+		logger.Info("ingest: TOC mode — parse→table of contents→persist→ready; skipping summarize/HyDE/multi-axis + table extraction")
 	} else if cfg.Ingest.Tables.Enabled {
 		logger.Info("ingest: pdf table extraction enabled",
 			"vertical_strategy", cfg.Ingest.Tables.VerticalStrategy,
@@ -540,7 +544,7 @@ func buildLLMFrom(c config.LLMConfig, provider, apiKey, baseURL, model string) (
 	}
 }
 
-func buildStrategy(c config.RetrievalConfig, client llmgate.Client, judge llmgate.Judge, store storage.Storage) retrieval.Strategy {
+func buildStrategy(c config.RetrievalConfig, client llmgate.Client, judge llmgate.Judge, store storage.Storage, pool *db.Pool) retrieval.Strategy {
 	switch c.Strategy {
 	case "judgewalk":
 		if judge == nil {
@@ -549,6 +553,8 @@ func buildStrategy(c config.RetrievalConfig, client llmgate.Client, judge llmgat
 		}
 		s := retrieval.NewJudgeWalkStrategy(judge)
 		s.PageLoader = storagePageLoader{s: store}
+		s.TOC = dbTOCProvider{db: pool}
+		s.Pages = storagePageStore{s: store}
 		return s
 	case "single-pass":
 		return retrieval.NewSinglePass(client)
@@ -680,3 +686,39 @@ func tableOptsFromConfig(c config.TablesConfig) *parser.TableOpts {
 		MinTableCols:       c.MinTableCols,
 	}
 }
+
+// storagePageStore serves the per-page text ingest persisted at
+// ingest.PagesKey, for judgewalk. Missing pages are not an error: the
+// strategy falls back to the section tree.
+type storagePageStore struct{ s storage.Storage }
+
+func (p storagePageStore) LoadPages(ctx context.Context, docID tree.DocumentID) ([]retrieval.NavPage, error) {
+	rc, _, err := p.s.Get(ctx, ingest.PagesKey(docID))
+	if err != nil {
+		return nil, err
+	}
+	defer rc.Close()
+	var pages []ingest.PageText
+	if err := json.NewDecoder(rc).Decode(&pages); err != nil {
+		return nil, err
+	}
+	out := make([]retrieval.NavPage, 0, len(pages))
+	for _, pg := range pages {
+		out = append(out, retrieval.NavPage{Number: pg.PageNumber, Text: pg.Text})
+	}
+	return out, nil
+}
+
+// dbTOCProvider reads documents.toc_tree for judgewalk and treewalk.
+type dbTOCProvider struct{ db *db.Pool }
+
+func (p dbTOCProvider) GetTOC(ctx context.Context, docID tree.DocumentID) ([]byte, error) {
+	doc, err := p.db.GetDocumentForWorker(ctx, docID)
+	if err != nil {
+		return nil, err
+	}
+	if len(doc.TOCTree) == 0 {
+		return nil, retrieval.ErrNoTOC
+	}
+	return doc.TOCTree, nil
+}

cmd/navbench/main.go

@@ -87,7 +87,12 @@ func main() {
 		fmt.Fprintln(os.Stderr, "judge:", err)
 		os.Exit(1)
 	}
-	lim := limit.New(limit.Config{Initial: 4, OnChange: func(e limit.Event) {
+	// Initial 16, not 4. A question issues 4-7 requests that do not
+	// depend on each other, so a limiter that starts at 4 serialises
+	// them into waves, and AIMD needs twenty successes to widen by one —
+	// slower than a 40-question run can recover from, especially after a
+	// transient failure halves it.
+	lim := limit.New(limit.Config{Initial: 16, OnChange: func(e limit.Event) {
 		fmt.Fprintf(os.Stderr, "  limiter %s %d -> %d %v\n", e.Cause, e.From, e.To, e.Err)
 	}})
 	nav := &retrieval.JudgeNavigator{Judge: retry.NewJudge(retry.Config{MaxRetries: 3})(limit.Judge(lim)(tj)), MaxLeaves: *maxLeaves, MaxPages: *maxPages}

cmd/server/main.go

@@ -15,6 +15,7 @@ package main
 import (
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -241,6 +242,8 @@ func run() error {
 	})
 	if cfg.Engine.Ingest.Mode == ingest.ModeMinimal {
 		logger.Info("ingest: MINIMAL mode — parse→persist→ready; skipping summarize/HyDE/multi-axis/TOC + table extraction")
+	} else if cfg.Engine.Ingest.Mode == ingest.ModeTOC {
+		logger.Info("ingest: TOC mode — parse→table of contents→persist→ready; skipping summarize/HyDE/multi-axis + table extraction")
 	} else if cfg.Engine.Ingest.Tables.Enabled {
 		logger.Info("ingest: pdf table extraction enabled",
 			"vertical_strategy", cfg.Engine.Ingest.Tables.VerticalStrategy,
@@ -488,7 +491,7 @@ func buildStrategy(c enginecfg.RetrievalConfig, client llmgate.Client, judge llm
 			log.Printf("retrieval: strategy judgewalk needs llm.judge configured; using treewalk")
 			return buildTreeWalkStrategy(c, client, store, pool)
 		}
-		return buildJudgeWalkStrategy(judge, store)
+		return buildJudgeWalkStrategy(judge, store, pool)
 	case "single-pass":
 		return retrieval.NewSinglePass(client)
 	case "chunked-tree":
@@ -535,7 +538,12 @@ func buildStrategySet(c enginecfg.RetrievalConfig, client llmgate.Client, judge
 		"auto":         retrieval.NewAuto(retrieval.NewSinglePass(client), buildTreeWalkStrategy(c, client, store, pool)),
 	}
 	if judge != nil {
-		set["judgewalk"] = buildJudgeWalkStrategy(judge, store)
+		set["judgewalk"] = buildJudgeWalkStrategy(judge, store, pool)
+	} else {
+		// The same fallback the default builder applies: a request that
+		// names judgewalk on a server with no Judge gets treewalk, not
+		// "unknown strategy".
+		set["judgewalk"] = set["treewalk"]
 	}
 	return set
 }
@@ -544,9 +552,13 @@ func buildStrategySet(c enginecfg.RetrievalConfig, client llmgate.Client, judge
 // leaves ranked in one request, the best sections' bodies ranked in one
 // or two more, no generative call (HAL-1371). Selectable per request as
 // strategy=judgewalk whenever llm.judge is configured.
-func buildJudgeWalkStrategy(judge llmgate.Judge, store storage.Storage) *retrieval.JudgeWalkStrategy {
+func buildJudgeWalkStrategy(judge llmgate.Judge, store storage.Storage, pool *db.Pool) *retrieval.JudgeWalkStrategy {
 	s := retrieval.NewJudgeWalkStrategy(judge)
 	s.PageLoader = storagePageLoader{s: store}
+	if pool != nil {
+		s.TOC = dbTOCProvider{db: pool}
+	}
+	s.Pages = storagePageStore{s: store}
 	return s
 }
 
@@ -687,3 +699,25 @@ func tableOptsFromConfig(c enginecfg.TablesConfig) *parser.TableOpts {
 		MinTableCols:       c.MinTableCols,
 	}
 }
+
+// storagePageStore serves the per-page text ingest persisted at
+// ingest.PagesKey, for judgewalk. Missing pages are not an error: the
+// strategy falls back to the section tree.
+type storagePageStore struct{ s storage.Storage }
+
+func (p storagePageStore) LoadPages(ctx context.Context, docID tree.DocumentID) ([]retrieval.NavPage, error) {
+	rc, _, err := p.s.Get(ctx, ingest.PagesKey(docID))
+	if err != nil {
+		return nil, err
+	}
+	defer rc.Close()
+	var pages []ingest.PageText
+	if err := json.NewDecoder(rc).Decode(&pages); err != nil {
+		return nil, err
+	}
+	out := make([]retrieval.NavPage, 0, len(pages))
+	for _, pg := range pages {
+		out = append(out, retrieval.NavPage{Number: pg.PageNumber, Text: pg.Text})
+	}
+	return out, nil
+}

config.example.yaml

@@ -362,6 +362,13 @@ ingest:
   #            summary-dependent strategies (chunked-tree, agentic)
   #            degrade to titles + raw content with no summaries.
   #
+  #   toc      parse -> build tree -> table of contents -> persist ->
+  #            ready. The page-based pipeline and nothing else: on a
+  #            Judge the TOC stage is three requests and seconds, and
+  #            page-based retrieval (treewalk, judgewalk) needs none of
+  #            the per-section enrichment. Table extraction skipped as
+  #            in minimal. The mode the FinanceBench evaluations use.
+  #
   # Override per-process with VLE_INGEST_MODE; on the deployed
   # vectorless-server use VLS_INGEST_MODE=minimal (no secret edit needed).
   mode: "full"

docs/evaluations/2026-09-19-head-to-head-chunk-and-embed.md

@@ -0,0 +1,91 @@
+# Vectorless on Jev against chunk-and-embed, on FinanceBench
+
+**Date:** 2026-09-19
+**Harness:** [vectorless-bench](https://github.com/hallelx2/vectorless-bench) `configs/financebench_jev.yaml` — the engine reached through the Python SDK, the baselines in-process
+**Corpus:** FinanceBench, 19 filings with questions among the 21 downloaded (68–549 pages), 40 questions, k=5, two repeats
+**Engine:** `cmd/engine --local`, `ingest.mode: toc`, `retrieval.strategy: judgewalk`, abstention off, caches off, Judge on, no generative call anywhere in retrieval
+**Question:** the post claims chunk-and-embed cannot be this fast and this exact. Is that measured?
+
+## Result
+
+The final run completed all 40 questions and both repeats with no
+errors and no engine restart. Every gold evidence page is among the
+returned pages for 34 of 40 questions; hit@5 (any returned unit holds
+the answer text) is 37 of 40; the answer text is in the first returned
+unit for 30 of 40. The six page misses are the four `navbench` had
+already found (three Boeing, one Pfizer) plus two where a gold page was
+one page off the returned one (Pfizer 70–71 → 71; Verizon 23 and 56 →
+23 and 57) — a page-boundary question for the full-page pass, not a
+navigation one.
+
+| system | how it retrieves | F1@5 | hit@5 | answer span in top-1 | p50 / query | $ / query | ingest, 19 filings | deterministic across repeats |
+|---|---|---|---|---|---|---|---|---|
+| **Vectorless, judgewalk on persisted pages, pages returned** (PR #68, final run 2026-09-21) | Jev ranks the TOC's sections, then page heads, then pages; the evidence pages are returned as-is, ahead of any section | 0.498 | **0.925** | **0.750** | 37 s (p95 75 s) | $0.0040 | 1,111 s (58 s / filing, sub-section splitting on) | 0.38 exact, 0.83 Jaccard |
+| Vectorless, judgewalk on persisted pages, sections mapped by page range (PR #68 before HAL-1390) | same navigation; the parser's sections covering the evidence pages returned | 0.197 | 0.475 | 0.000 | 29 s | $0.0040 | — | — |
+| Vectorless, judgewalk on the section tree (first pass) | same navigation over the parser's sections and their bodies | 0.453 | 0.650 | 0.575 | 50 s | $0.0067 | 1,080 s (57 s / filing) | 0.43 exact, 0.69 Jaccard |
+| chunk-and-embed, BGE-small | 512-token chunks, bge-small-en-v1.5 on the CPU, cosine top-5 | 0.170 | 0.375 | 0.225 | 49 ms | $0 | 1,972 s (104 s / filing, 4 threads) | 1.00 |
+| BM25 | 512-token chunks, BM25 top-5 | 0.096 | 0.200 | 0.075 | 48 ms | $0 | 2 s | 1.00 |
+
+Scoring: a question's gold is FinanceBench's evidence text and answer; a
+returned unit is a hit when it contains the gold span (numbers matched
+as numbers). F1@5 is the harness's primary quality; "answer span in
+top-1" is whether the first returned unit contains it.
+
+## What the runs taught, in order
+
+The server's judgewalk was navigating the **parser's section tree** over
+section bodies. The Jev-built table of contents was persisted for
+treewalk alone, and per-page text was never persisted. That scored
+hit@5 0.65 where the same navigation over real pages had scored 0.90
+in `navbench` — the parser's page attribution is exactly what HAL-1375
+had shown to be unreliable. Ingest now persists the pages beside
+`documents.toc_tree`, and judgewalk navigates the persisted TOC over
+them (PR #68).
+
+That alone made things **worse**: hit@5 0.475, the answer in the first
+result never. The right pages were found and then mapped back to the
+parser's sections covering them by page range — the same unreliable
+attribution, one step later. Page-based retrieval now returns its
+pages (HAL-1390): `/v1/query` leads with the evidence pages as units
+of their own, `page` and `confidence` set. On the full 40, hit@5 went
+0.475 → 0.925 and the answer in the first unit 0.000 → 0.750.
+
+Three more things the run surfaced, each fixed on the way:
+
+- `/v1/query` dropped retrieval's usage, so a client benchmarking
+  retrieval alone saw $0 (PR #67).
+- `ingest.mode: minimal` skips the TOC stage entirely; a `toc` mode now
+  runs the page-based pipeline and nothing else (PR #67).
+- The SDK rejected an abstained response for lacking a model name
+  (vectorless-sdk #3). Abstention is off for this run so judgewalk's
+  low-confidence best guess is scored rather than blanked.
+
+## What the numbers say, and what they do not
+
+- **Latency.** Chunk-and-embed answers a query in 50 ms; Vectorless on
+  Jev takes tens of seconds, all of it Judge round-trips. The post's
+  speed claim is about *ingest* and about the *generative* path it
+  replaced, not about beating a cosine lookup — say so.
+- **Ingest.** Embedding 19 filings with a small model on four CPU
+  threads took 33 minutes; the Jev TOC stage took 18 minutes for the
+  same filings with sub-section splitting on, for $0.11. With a GPU the
+  embedding number collapses; with more Jev concurrency so does ours.
+- **Exactness.** Even on the section tree, Vectorless put the answer's
+  text in its first result 2.6× as often as BGE-small and 7.7× as often
+  as BM25. That is the citation claim, and it is measured.
+- **Determinism.** The baselines are exact across repeats; judgewalk is
+  not (0.43 exact-match of returned sets), because Jev's probabilities
+  near the threshold move between calls. A product wants this number
+  reported, not hidden.
+- **Not measured here:** answer quality from the retrieved context
+  (HAL-1387), the tree-walk chat loop on the same questions, Gemini
+  embeddings (free-tier quota), and anything on a GPU.
+
+## Reproduce
+
+```bash
+# engine, local mode, Judge on, page-based pipeline only
+VLE_INGEST_MODE=toc TYPESAFE_API_KEY=… ./engine --local -config config.yaml   # retrieval.strategy: judgewalk, abstain off, cache off
+# bench
+cd vectorless-bench && VECTORLESS_BASE_URL=http://localhost:7654 vlbench run --config configs/financebench_jev.yaml
+```

docs/evaluations/2026-09-19-leaf-granularity.md

@@ -62,6 +62,37 @@ Two sources, in order of trust, both "select, don't generate":
   Operations" to "Contingent Obligations"; its Item 8 became the six
   statements and twenty-one notes.
 
+## Recursive splitting: a negative result (2026-09-25)
+
+The splitter ran once over the leaves the contents pass produced, so a
+70-page Item 8 split into its notes and a 28-page Note 1 with its own
+headings stayed whole. Making it descend through each generation —
+same two sources, same per-leaf Judge confirmation, depth-capped —
+was built, tested, and measured against the single-pass trees:
+
+| | one generation | recursive |
+|---|---|---|
+| leaves per filing, median | 69 | 72 |
+| median leaf span | 1 p | 1 p |
+| leaf span, p90 | 7 p | 9 p |
+| max tree depth | 3 | 4 |
+| gold pages inside a leaf | 47 / 47 | 47 / 47 |
+| span of the leaf holding a gold page, median | **5 p** | 6 p |
+| right section chosen | 40 / 40 | 40 / 40 |
+| every gold page in the evidence | **36 / 40** | 34 / 40 |
+| pages read / question | 40.6 | 40.8 |
+| ingest: Judge requests / cost / wall | **278 / $0.13 / 489 s** | 470 / $0.21 / 1,174 s |
+
+Nothing improved and ingest paid 1.7× the requests and 2.4× the wall
+clock. The reason is visible in the first two rows: after one pass the
+median leaf is already a single page, because the per-leaf cap allows
+one sub-leaf per two pages. The leaves that remain large are the ones
+the splitter could not find headings in — and a second look at the
+same text with the same rules does not find them either.
+
+`SplitGenerations` keeps the capability for a document unlike a 10-K,
+defaulting to one. The lesson generalises: recursion is not granularity.
+
 ## What is still missed
 
 Four questions, the same four as before the split: Boeing's legal