From 29fe58fe3a8f9d75479de68c3e71dfef5c95a3e7 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Sat, 19 Sep 2026 09:46:35 +0100 Subject: [PATCH 01/15] fix(config): accept retrieval.strategy judgewalk --- pkg/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index b334469..f7c1676 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1319,7 +1319,7 @@ func (c Config) Validate() error { } switch c.Retrieval.Strategy { - case "auto", "single-pass", "chunked-tree", "agentic", "treewalk": + case "auto", "single-pass", "chunked-tree", "agentic", "treewalk", "judgewalk": default: return fmt.Errorf("unknown retrieval.strategy: %q", c.Retrieval.Strategy) } From 771f487b36f5355bc863cdf7bb2eb1850f77c963 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Sat, 19 Sep 2026 10:00:15 +0100 Subject: [PATCH 02/15] =?UTF-8?q?feat(ingest):=20toc=20mode=20=E2=80=94=20?= =?UTF-8?q?parse,=20table=20of=20contents=20on=20the=20Judge,=20persist,?= =?UTF-8?q?=20ready?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Minimal mode skips the TOC stage, so a minimal-mode document reaches judgewalk with the raw parser tree instead of the Jev-built table of contents the evaluations measured; full mode adds minutes of per-section generative enrichment that page-based retrieval never reads. toc mode is the page-based pipeline and nothing else. Found standing up the FinanceBench head-to-head. --- cmd/engine/main.go | 2 ++ cmd/server/main.go | 2 ++ config.example.yaml | 7 +++++++ pkg/config/config.go | 4 ++-- pkg/ingest/ingest.go | 25 ++++++++++++++++++----- pkg/ingest/minimal_mode_test.go | 35 +++++++++++++++++++++++++++++++++ 6 files changed, 68 insertions(+), 7 deletions(-) diff --git a/cmd/engine/main.go b/cmd/engine/main.go index dc126d6..3b26287 100644 --- a/cmd/engine/main.go +++ b/cmd/engine/main.go @@ -249,6 +249,8 @@ func run() error { }) if cfg.Ingest.Mode == ingest.ModeMinimal { logger.Info("ingest: MINIMAL mode — parse→persist→ready; skipping summarize/HyDE/multi-axis/TOC + table extraction") + } else if cfg.Ingest.Mode == ingest.ModeTOC { + logger.Info("ingest: TOC mode — parse→table of contents→persist→ready; skipping summarize/HyDE/multi-axis + table extraction") } else if cfg.Ingest.Tables.Enabled { logger.Info("ingest: pdf table extraction enabled", "vertical_strategy", cfg.Ingest.Tables.VerticalStrategy, diff --git a/cmd/server/main.go b/cmd/server/main.go index 48fe422..9feb16e 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -241,6 +241,8 @@ func run() error { }) if cfg.Engine.Ingest.Mode == ingest.ModeMinimal { logger.Info("ingest: MINIMAL mode — parse→persist→ready; skipping summarize/HyDE/multi-axis/TOC + table extraction") + } else if cfg.Engine.Ingest.Mode == ingest.ModeTOC { + logger.Info("ingest: TOC mode — parse→table of contents→persist→ready; skipping summarize/HyDE/multi-axis + table extraction") } else if cfg.Engine.Ingest.Tables.Enabled { logger.Info("ingest: pdf table extraction enabled", "vertical_strategy", cfg.Engine.Ingest.Tables.VerticalStrategy, diff --git a/config.example.yaml b/config.example.yaml index be9e3f3..5336fd6 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -362,6 +362,13 @@ ingest: # summary-dependent strategies (chunked-tree, agentic) # degrade to titles + raw content with no summaries. # + # toc parse -> build tree -> table of contents -> persist -> + # ready. The page-based pipeline and nothing else: on a + # Judge the TOC stage is three requests and seconds, and + # page-based retrieval (treewalk, judgewalk) needs none of + # the per-section enrichment. Table extraction skipped as + # in minimal. The mode the FinanceBench evaluations use. + # # Override per-process with VLE_INGEST_MODE; on the deployed # vectorless-server use VLS_INGEST_MODE=minimal (no secret edit needed). mode: "full" diff --git a/pkg/config/config.go b/pkg/config/config.go index f7c1676..d9b5cdd 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1339,9 +1339,9 @@ func (c Config) Validate() error { } switch c.Ingest.Mode { - case "", "full", "minimal": + case "", "full", "minimal", "toc": default: - return fmt.Errorf("ingest.mode must be one of full|minimal, got %q", c.Ingest.Mode) + return fmt.Errorf("ingest.mode must be one of full|minimal|toc, got %q", c.Ingest.Mode) } if c.Ingest.HyDE.NumQuestions < 0 { diff --git a/pkg/ingest/ingest.go b/pkg/ingest/ingest.go index 4d2b6e0..8462fa7 100644 --- a/pkg/ingest/ingest.go +++ b/pkg/ingest/ingest.go @@ -50,6 +50,14 @@ import ( // and table extraction. Any other value runs the full pipeline. const ModeMinimal = "minimal" +// ModeTOC is parse → build tree → table of contents → persist → ready: +// the page-based pipeline and nothing else. On a Judge the TOC stage is +// three requests and seconds; the per-section generative enrichment +// (summarize, HyDE, multi-axis) that full mode adds is minutes and is +// not used by page-based retrieval. Table extraction is skipped as in +// minimal mode: the page text still carries the tables' text. +const ModeTOC = "toc" + // docPersister is the narrow slice of *db.Pool the parse → persist → // ready path depends on. Declaring it here (rather than threading the // concrete *db.Pool) lets the minimal-mode runner be exercised with a @@ -377,7 +385,7 @@ func (p *Pipeline) Handler() queue.Handler { // tree → persist → ready, with no LLM enrichment and no table // extraction. Otherwise it runs the full enrichment pipeline below. func (p *Pipeline) Run(ctx context.Context, pl Payload) error { - if p.Mode == ModeMinimal { + if p.Mode == ModeMinimal || p.Mode == ModeTOC { return p.runMinimal(ctx, p.DB, pl) } @@ -730,13 +738,20 @@ func (p *Pipeline) runMinimal(ctx context.Context, store docPersister, pl Payloa return err } - // Skip summarize / HyDE / multi-axis / TOC entirely — flip straight - // to ready. The document is now queryable via the page-based - // strategy (synthesised TOC + raw page reads). + // Minimal mode skips summarize / HyDE / multi-axis / TOC entirely + // and flips straight to ready; the document is queryable via the + // page-based strategy on a TOC synthesised from the section tree. + // TOC mode builds the real table of contents first — same builder + // and persistence as full mode, non-fatal for the same reason. + if p.Mode == ModeTOC && pl.ContentType == "application/pdf" { + if err := p.runTOCBuilder(ctx, pl.DocumentID, parsed, log); err != nil { + log.Warn("ingest: toc-builder failed; falling back to NULL toc_tree", "err", err) + } + } if err := store.SetDocumentStatus(ctx, pl.DocumentID, db.StatusReady, ""); err != nil { return err } - log.Info("ingest: ready (minimal mode)") + log.Info("ingest: ready (" + p.Mode + " mode)") return nil } diff --git a/pkg/ingest/minimal_mode_test.go b/pkg/ingest/minimal_mode_test.go index e753b1b..c5940fb 100644 --- a/pkg/ingest/minimal_mode_test.go +++ b/pkg/ingest/minimal_mode_test.go @@ -315,3 +315,38 @@ func reconstructTree(_ tree.DocumentID, title string, rows []db.Section) *tree.S return &tree.Section{Title: title, Children: topLevel} } } + +// TOC mode on a non-PDF is minimal mode: nothing for the TOC builder to +// do, no model call, ready. +func TestTOCModeOnMarkdownMakesNoLLMCall(t *testing.T) { + ctx := context.Background() + st, err := storage.NewLocal(t.TempDir()) + if err != nil { + t.Fatal(err) + } + body := []byte("# Title\n\n## Section A\n\nAlpha.\n\n## Section B\n\nBeta.\n") + docID := NewDocumentID() + srcKey := SourceKey(docID, "doc.md") + if err := st.Put(ctx, srcKey, bytes.NewReader(body), storage.Metadata{ContentType: "text/markdown", Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + p := NewPipeline(Pipeline{ + Storage: st, + LLM: &failIfCalledLLM{t: t}, + Parsers: DefaultRegistry(), + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + Mode: ModeTOC, + TOCEnabled: true, + }) + fake := &fakeDocStore{} + if err := p.runMinimal(ctx, fake, Payload{DocumentID: docID, ContentType: "text/markdown", Filename: "doc.md", SourceRef: srcKey}); err != nil { + t.Fatalf("run: %v", err) + } + status, errMsg, sections := fake.snapshot() + if status != db.StatusReady { + t.Fatalf("status %q (%q) want ready", status, errMsg) + } + if len(sections) == 0 { + t.Fatal("no sections persisted") + } +} From 6c977b3b19a4cc28b557f07c85e63af00f066490 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Sat, 19 Sep 2026 10:11:46 +0100 Subject: [PATCH 03/15] fix(api): /v1/query reports usage and never an empty model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retrieval's Usage was accumulated and dropped on /v1/query, so a client benchmarking retrieval alone saw $0 and zero calls; /v1/answer had always reported it. The response now carries usage with the same keys, and model falls back to the strategy name when the request named none — a Judge-navigated query need not — including on abstention, whose response had no model field at all and failed the SDK's schema. --- internal/api/abstention_test.go | 4 ++-- internal/api/server.go | 32 ++++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/internal/api/abstention_test.go b/internal/api/abstention_test.go index 2018cdf..48ed4d6 100644 --- a/internal/api/abstention_test.go +++ b/internal/api/abstention_test.go @@ -198,7 +198,7 @@ func TestRespondAbstained(t *testing.T) { confidences := map[tree.SectionID]float64{"sec_a": 0.12, "sec_b": 0.30} rec := httptest.NewRecorder() - d.respondAbstained(rec, tree.DocumentID("doc_x"), "what is x?", confidences, nil) + d.respondAbstained(rec, tree.DocumentID("doc_x"), "what is x?", "", confidences, nil, retrieval.Usage{}) if rec.Code != http.StatusOK { t.Fatalf("status = %d, want 200", rec.Code) @@ -279,7 +279,7 @@ func TestRespondAbstainedTraceTokenAbsent(t *testing.T) { Abstain: config.AbstainBlock{Enabled: true, Below: 0.4}, } rec := httptest.NewRecorder() - d.respondAbstained(rec, tree.DocumentID("doc_x"), "q", map[tree.SectionID]float64{"a": 0.1}, nil) + d.respondAbstained(rec, tree.DocumentID("doc_x"), "q", "", map[tree.SectionID]float64{"a": 0.1}, nil, retrieval.Usage{}) var body map[string]any _ = json.Unmarshal(rec.Body.Bytes(), &body) diff --git a/internal/api/server.go b/internal/api/server.go index c619488..b8a2c6e 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -601,12 +601,19 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { started := time.Now() plan, _ := d.runPlanner(r.Context(), body.Query, body.EnablePlanning) - ids, confidences, err := d.runSelection(r.Context(), t, plan, body.Query, budget) + ids, confidences, selUsage, err := d.runSelectionWithUsage(r.Context(), t, plan, body.Query, budget) if err != nil { d.Logger.Error("query: strategy failed", "err", err, "document_id", body.DocumentID) writeErr(w, http.StatusInternalServerError, "retrieval failed: "+err.Error()) return } + // The model the caller named, or — when it named none, as a + // Judge-navigated query need not — the strategy, so the field is + // never empty and a client can always tell what answered. + modelUsed := body.Model + if modelUsed == "" { + modelUsed = d.Strategy.Name() + } // Phase 2.4 abstention: if every confident pick is below the // configured threshold, refuse to ground an answer in evidence @@ -615,7 +622,7 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { // responses (no confidences) always fall through to the normal // path so older models keep working. if d.abstentionEnabled(body.EnableAbstain) && shouldAbstain(confidences, d.Abstain.Below) { - d.respondAbstained(w, body.DocumentID, body.Query, confidences, plan) + d.respondAbstained(w, body.DocumentID, body.Query, modelUsed, confidences, plan, selUsage) return } @@ -678,10 +685,13 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { "document_id": body.DocumentID, "query": body.Query, "strategy": d.Strategy.Name(), - "model": body.Model, + "model": modelUsed, "sections": sections, "elapsed_ms": time.Since(started).Milliseconds(), "trace_token": traceToken, + // What retrieval cost. /v1/answer always reported this; /v1/query + // dropped it, so a caller benchmarking retrieval alone saw $0. + "usage": usageJSON(selUsage), } if plan != nil { resp["plan"] = plan @@ -1650,11 +1660,13 @@ const abstentionAnswerText = "I cannot answer this question from the supplied do // the response in the replay log because there's no meaningful // retrieval result to reproduce. Callers replaying an abstention // will simply re-run /v1/query. -func (d Deps) respondAbstained(w http.ResponseWriter, docID tree.DocumentID, query string, confidences map[tree.SectionID]float64, plan *retrieval.Plan) { +func (d Deps) respondAbstained(w http.ResponseWriter, docID tree.DocumentID, query string, model string, confidences map[tree.SectionID]float64, plan *retrieval.Plan, usage retrieval.Usage) { resp := map[string]any{ "document_id": docID, "query": query, "strategy": d.Strategy.Name(), + "model": model, + "usage": usageJSON(usage), "sections": []any{}, "abstained": true, "abstention_reason": abstentionReason, @@ -1667,6 +1679,18 @@ func (d Deps) respondAbstained(w http.ResponseWriter, docID tree.DocumentID, que writeJSON(w, http.StatusOK, resp) } +// usageJSON is the wire shape of retrieval.Usage on /v1/query, the same +// keys /v1/answer uses for total_usage. +func usageJSON(u retrieval.Usage) map[string]any { + return map[string]any{ + "input_tokens": u.InputTokens, + "output_tokens": u.OutputTokens, + "total_tokens": u.TotalTokens, + "cost_usd": u.CostUSD, + "llm_calls": u.LLMCalls, + } +} + // respondAbstainedAnswer writes the abstention shape for /v1/answer. // The answer text is the canonical refusal; citations is empty; // usage carries the LLM tokens spent up to the abstention point From f3794c4d0354eff885457d3fa3225363220bdb37 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Sat, 19 Sep 2026 11:09:04 +0100 Subject: [PATCH 04/15] review: judgewalk in the per-request strategy set falls back to treewalk without a Judge; validator test lists judgewalk --- cmd/server/main.go | 5 +++++ pkg/config/config_test.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 9feb16e..905c93c 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -538,6 +538,11 @@ func buildStrategySet(c enginecfg.RetrievalConfig, client llmgate.Client, judge } if judge != nil { set["judgewalk"] = buildJudgeWalkStrategy(judge, store) + } else { + // The same fallback the default builder applies: a request that + // names judgewalk on a server with no Judge gets treewalk, not + // "unknown strategy". + set["judgewalk"] = set["treewalk"] } return set } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 9a3bb34..7761240 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -652,7 +652,7 @@ func TestValidateLLMDrivers(t *testing.T) { func TestValidateRetrievalStrategy(t *testing.T) { t.Parallel() - for _, s := range []string{"auto", "single-pass", "chunked-tree", "agentic", "treewalk"} { + for _, s := range []string{"auto", "single-pass", "chunked-tree", "agentic", "treewalk", "judgewalk"} { cfg := Default() cfg.Database.URL = "postgres://localhost/test" cfg.Retrieval.Strategy = s From d93dd523e270bca204942d6a29069d94de1c34c4 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Sat, 19 Sep 2026 11:25:35 +0100 Subject: [PATCH 05/15] review: /v1/query usage accumulates planner, selection and re-rank, as /v1/answer does --- internal/api/server.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/internal/api/server.go b/internal/api/server.go index b8a2c6e..2660d75 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -600,13 +600,19 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { started := time.Now() - plan, _ := d.runPlanner(r.Context(), body.Query, body.EnablePlanning) + // Usage accumulates the way /v1/answer's does: planner, selection, + // re-rank. Span extraction is per section and not counted on either + // endpoint today. + totalUsage := retrieval.Usage{} + plan, planUsage := d.runPlanner(r.Context(), body.Query, body.EnablePlanning) + totalUsage.Add(planUsage) ids, confidences, selUsage, err := d.runSelectionWithUsage(r.Context(), t, plan, body.Query, budget) if err != nil { d.Logger.Error("query: strategy failed", "err", err, "document_id", body.DocumentID) writeErr(w, http.StatusInternalServerError, "retrieval failed: "+err.Error()) return } + totalUsage.Add(selUsage) // The model the caller named, or — when it named none, as a // Judge-navigated query need not — the strategy, so the field is // never empty and a client can always tell what answered. @@ -622,7 +628,7 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { // responses (no confidences) always fall through to the normal // path so older models keep working. if d.abstentionEnabled(body.EnableAbstain) && shouldAbstain(confidences, d.Abstain.Below) { - d.respondAbstained(w, body.DocumentID, body.Query, modelUsed, confidences, plan, selUsage) + d.respondAbstained(w, body.DocumentID, body.Query, modelUsed, confidences, plan, totalUsage) return } @@ -654,7 +660,9 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { // never drop sections — at worst the strategy's order is // preserved (see retrieval.ReRanker.ReRank). if d.reRankEnabled(body.EnableReRank) { - enriched, _ = d.runReRank(r.Context(), enriched, body.Query, body.Model) + var reRankUsage retrieval.Usage + enriched, reRankUsage = d.runReRank(r.Context(), enriched, body.Query, body.Model) + totalUsage.Add(reRankUsage) } // Optional: per-section answer-span extraction. Opt-in via config — @@ -691,7 +699,7 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { "trace_token": traceToken, // What retrieval cost. /v1/answer always reported this; /v1/query // dropped it, so a caller benchmarking retrieval alone saw $0. - "usage": usageJSON(selUsage), + "usage": usageJSON(totalUsage), } if plan != nil { resp["plan"] = plan From 1613d31421984904da2f8821e655c8814767be11 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Sat, 19 Sep 2026 11:47:15 +0100 Subject: [PATCH 06/15] feat: judgewalk navigates the persisted table of contents over persisted pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The server's judgewalk navigated the parser's section tree and read section bodies, while the Jev-built table of contents was persisted for treewalk alone and per-page text was never persisted at all. On the FinanceBench head-to-head that scored hit@5 0.65 where the same navigation over real pages scored 0.90 (navbench) — the parser's page attribution is what HAL-1375 showed to be unreliable. Ingest now persists the pages it built the table of contents from, as JSON at ingest.PagesKey beside documents.toc_tree. JudgeWalkStrategy takes a TOCProvider and a PageStore; with both it navigates the TOC's leaves (sub-sections and all, with real page ranges) over the persisted pages, and answers the API in the sections that cover the evidence pages with those pages cited. Without either it falls back to the section tree as before. Both binaries wire the providers. --- cmd/engine/main.go | 44 +++++++++++- cmd/server/main.go | 33 ++++++++- pkg/ingest/ingest.go | 25 +++++++ pkg/ingest/minimal_mode_test.go | 28 ++++++++ pkg/ingest/toc_builder.go | 4 +- pkg/retrieval/judgewalk.go | 121 +++++++++++++++++++++++++++++++- pkg/retrieval/judgewalk_test.go | 58 +++++++++++++++ 7 files changed, 305 insertions(+), 8 deletions(-) diff --git a/cmd/engine/main.go b/cmd/engine/main.go index 3b26287..74a22b4 100644 --- a/cmd/engine/main.go +++ b/cmd/engine/main.go @@ -7,6 +7,7 @@ package main import ( "context" "crypto/tls" + "encoding/json" "errors" "flag" "fmt" @@ -36,6 +37,7 @@ import ( "github.com/hallelx2/vectorless-engine/pkg/queue" "github.com/hallelx2/vectorless-engine/pkg/retrieval" "github.com/hallelx2/vectorless-engine/pkg/storage" + "github.com/hallelx2/vectorless-engine/pkg/tree" ) // version is set at build time via -ldflags "-X main.version=..." @@ -141,7 +143,7 @@ func run() error { } else { logger.Warn("judge: none configured — TOC judgements run on the generative driver, one call per page (set TYPESAFE_API_KEY)") } - strategy := buildStrategy(cfg.Retrieval, llmClient, judge, store) + strategy := buildStrategy(cfg.Retrieval, llmClient, judge, store, pool) // Wrap with caching if enabled. if cfg.Retrieval.Cache.Enabled { @@ -542,7 +544,7 @@ func buildLLMFrom(c config.LLMConfig, provider, apiKey, baseURL, model string) ( } } -func buildStrategy(c config.RetrievalConfig, client llmgate.Client, judge llmgate.Judge, store storage.Storage) retrieval.Strategy { +func buildStrategy(c config.RetrievalConfig, client llmgate.Client, judge llmgate.Judge, store storage.Storage, pool *db.Pool) retrieval.Strategy { switch c.Strategy { case "judgewalk": if judge == nil { @@ -551,6 +553,8 @@ func buildStrategy(c config.RetrievalConfig, client llmgate.Client, judge llmgat } s := retrieval.NewJudgeWalkStrategy(judge) s.PageLoader = storagePageLoader{s: store} + s.TOC = dbTOCProvider{db: pool} + s.Pages = storagePageStore{s: store} return s case "single-pass": return retrieval.NewSinglePass(client) @@ -682,3 +686,39 @@ func tableOptsFromConfig(c config.TablesConfig) *parser.TableOpts { MinTableCols: c.MinTableCols, } } + +// storagePageStore serves the per-page text ingest persisted at +// ingest.PagesKey, for judgewalk. Missing pages are not an error: the +// strategy falls back to the section tree. +type storagePageStore struct{ s storage.Storage } + +func (p storagePageStore) LoadPages(ctx context.Context, docID tree.DocumentID) ([]retrieval.NavPage, error) { + rc, _, err := p.s.Get(ctx, ingest.PagesKey(docID)) + if err != nil { + return nil, err + } + defer rc.Close() + var pages []ingest.PageText + if err := json.NewDecoder(rc).Decode(&pages); err != nil { + return nil, err + } + out := make([]retrieval.NavPage, 0, len(pages)) + for _, pg := range pages { + out = append(out, retrieval.NavPage{Number: pg.PageNumber, Text: pg.Text}) + } + return out, nil +} + +// dbTOCProvider reads documents.toc_tree for judgewalk and treewalk. +type dbTOCProvider struct{ db *db.Pool } + +func (p dbTOCProvider) GetTOC(ctx context.Context, docID tree.DocumentID) ([]byte, error) { + doc, err := p.db.GetDocumentForWorker(ctx, docID) + if err != nil { + return nil, err + } + if len(doc.TOCTree) == 0 { + return nil, retrieval.ErrNoTOC + } + return doc.TOCTree, nil +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 905c93c..b6f1728 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -15,6 +15,7 @@ package main import ( "context" "crypto/tls" + "encoding/json" "errors" "flag" "fmt" @@ -490,7 +491,7 @@ func buildStrategy(c enginecfg.RetrievalConfig, client llmgate.Client, judge llm log.Printf("retrieval: strategy judgewalk needs llm.judge configured; using treewalk") return buildTreeWalkStrategy(c, client, store, pool) } - return buildJudgeWalkStrategy(judge, store) + return buildJudgeWalkStrategy(judge, store, pool) case "single-pass": return retrieval.NewSinglePass(client) case "chunked-tree": @@ -537,7 +538,7 @@ func buildStrategySet(c enginecfg.RetrievalConfig, client llmgate.Client, judge "auto": retrieval.NewAuto(retrieval.NewSinglePass(client), buildTreeWalkStrategy(c, client, store, pool)), } if judge != nil { - set["judgewalk"] = buildJudgeWalkStrategy(judge, store) + set["judgewalk"] = buildJudgeWalkStrategy(judge, store, pool) } else { // The same fallback the default builder applies: a request that // names judgewalk on a server with no Judge gets treewalk, not @@ -551,9 +552,13 @@ func buildStrategySet(c enginecfg.RetrievalConfig, client llmgate.Client, judge // leaves ranked in one request, the best sections' bodies ranked in one // or two more, no generative call (HAL-1371). Selectable per request as // strategy=judgewalk whenever llm.judge is configured. -func buildJudgeWalkStrategy(judge llmgate.Judge, store storage.Storage) *retrieval.JudgeWalkStrategy { +func buildJudgeWalkStrategy(judge llmgate.Judge, store storage.Storage, pool *db.Pool) *retrieval.JudgeWalkStrategy { s := retrieval.NewJudgeWalkStrategy(judge) s.PageLoader = storagePageLoader{s: store} + if pool != nil { + s.TOC = dbTOCProvider{db: pool} + } + s.Pages = storagePageStore{s: store} return s } @@ -694,3 +699,25 @@ func tableOptsFromConfig(c enginecfg.TablesConfig) *parser.TableOpts { MinTableCols: c.MinTableCols, } } + +// storagePageStore serves the per-page text ingest persisted at +// ingest.PagesKey, for judgewalk. Missing pages are not an error: the +// strategy falls back to the section tree. +type storagePageStore struct{ s storage.Storage } + +func (p storagePageStore) LoadPages(ctx context.Context, docID tree.DocumentID) ([]retrieval.NavPage, error) { + rc, _, err := p.s.Get(ctx, ingest.PagesKey(docID)) + if err != nil { + return nil, err + } + defer rc.Close() + var pages []ingest.PageText + if err := json.NewDecoder(rc).Decode(&pages); err != nil { + return nil, err + } + out := make([]retrieval.NavPage, 0, len(pages)) + for _, pg := range pages { + out = append(out, retrieval.NavPage{Number: pg.PageNumber, Text: pg.Text}) + } + return out, nil +} diff --git a/pkg/ingest/ingest.go b/pkg/ingest/ingest.go index 8462fa7..41a4869 100644 --- a/pkg/ingest/ingest.go +++ b/pkg/ingest/ingest.go @@ -470,6 +470,14 @@ func (p *Pipeline) runTOCBuilder(ctx context.Context, docID tree.DocumentID, par log.Info("ingest: toc-builder skipped; no per-page text available") return nil } + // The pages are the ground truth every page-level stage reasons + // over (HAL-1375). Persist them beside the table of contents so + // retrieval navigates the same text ingest did, not a per-section + // reconstruction of it. Non-fatal: without them judgewalk falls + // back to the section tree. + if err := p.persistPages(ctx, docID, pages); err != nil { + log.Warn("ingest: pages not persisted; judgewalk will use the section tree", "err", err) + } model := p.TOCModel if model == "" { model = p.SummaryModel @@ -1297,6 +1305,23 @@ func NewDocumentID() tree.DocumentID { // SourceKey returns the canonical storage key where an ingest payload's // original bytes live. +// PagesKey is where a document's per-page text lives in storage: a +// JSON array of {page_number, text}, written by ingest for paged +// documents and read by page-based retrieval. +func PagesKey(id tree.DocumentID) string { return "pages/" + string(id) + ".json" } + +// persistPages writes the per-page text to storage at PagesKey. +func (p *Pipeline) persistPages(ctx context.Context, docID tree.DocumentID, pages []PageText) error { + if p.Storage == nil { + return fmt.Errorf("no storage") + } + raw, err := json.Marshal(pages) + if err != nil { + return err + } + return p.Storage.Put(ctx, PagesKey(docID), bytes.NewReader(raw), storage.Metadata{ContentType: "application/json", Size: int64(len(raw))}) +} + func SourceKey(id tree.DocumentID, filename string) string { // Keep the original extension so future content-type sniffing works. ext := path.Ext(filename) diff --git a/pkg/ingest/minimal_mode_test.go b/pkg/ingest/minimal_mode_test.go index c5940fb..6e4f1e9 100644 --- a/pkg/ingest/minimal_mode_test.go +++ b/pkg/ingest/minimal_mode_test.go @@ -3,6 +3,7 @@ package ingest import ( "bytes" "context" + "encoding/json" "io" "log/slog" "os" @@ -350,3 +351,30 @@ func TestTOCModeOnMarkdownMakesNoLLMCall(t *testing.T) { t.Fatal("no sections persisted") } } + +// Pages are persisted beside the table of contents as JSON at PagesKey, +// in the shape page-based retrieval reads back. +func TestPersistPagesRoundTrip(t *testing.T) { + ctx := context.Background() + st, err := storage.NewLocal(t.TempDir()) + if err != nil { + t.Fatal(err) + } + p := &Pipeline{Storage: st} + pages := []PageText{{PageNumber: 1, Text: "cover"}, {PageNumber: 3, Text: "Item 1. Business"}} + if err := p.persistPages(ctx, "doc_x", pages); err != nil { + t.Fatal(err) + } + rc, _, err := st.Get(ctx, PagesKey("doc_x")) + if err != nil { + t.Fatal(err) + } + defer rc.Close() + var back []PageText + if err := json.NewDecoder(rc).Decode(&back); err != nil { + t.Fatal(err) + } + if len(back) != 2 || back[1].PageNumber != 3 || back[1].Text != "Item 1. Business" { + t.Errorf("round trip: %+v", back) + } +} diff --git a/pkg/ingest/toc_builder.go b/pkg/ingest/toc_builder.go index 2f404e6..bf6784a 100644 --- a/pkg/ingest/toc_builder.go +++ b/pkg/ingest/toc_builder.go @@ -23,8 +23,8 @@ import ( // pages produced by the existing parser pipeline and over synthetic // fixtures used in tests. type PageText struct { - PageNumber int - Text string + PageNumber int `json:"page_number"` + Text string `json:"text"` } // TOCBuilder builds an LLM-derived table-of-contents tree for a diff --git a/pkg/retrieval/judgewalk.go b/pkg/retrieval/judgewalk.go index a8be4f8..64a19f0 100644 --- a/pkg/retrieval/judgewalk.go +++ b/pkg/retrieval/judgewalk.go @@ -2,6 +2,7 @@ package retrieval import ( "context" + "encoding/json" "fmt" "sort" "strings" @@ -561,6 +562,22 @@ func judgeUsage(res *llmgate.Judgment) Usage { type JudgeWalkStrategy struct { Navigator JudgeNavigator PageLoader PageContentLoader + + // TOC and Pages, when both are set and both have data for the + // document, are what navigation runs over: the table of contents + // ingest built (leaves with real page ranges, sub-sections and all) + // and the per-page text ingest persisted beside it. That is the + // pipeline the FinanceBench evaluations measured. Without either, + // navigation falls back to the section tree and section bodies — + // the parser's page attribution, which is what HAL-1375 showed to + // be unreliable — and scored 0.65 where the pages scored 0.90. + TOC TOCProvider + Pages PageStore +} + +// PageStore serves a document's per-page text, as ingest persisted it. +type PageStore interface { + LoadPages(ctx context.Context, docID tree.DocumentID) ([]NavPage, error) } const strategyNameJudgeWalk = "judgewalk" @@ -582,10 +599,112 @@ func (s *JudgeWalkStrategy) Select(ctx context.Context, t *tree.Tree, query stri } // SelectWithCost runs navigation and reports usage. -func (s *JudgeWalkStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, query string, _ ContextBudget) (*Result, error) { +func (s *JudgeWalkStrategy) SelectWithCost(ctx context.Context, t *tree.Tree, query string, budget ContextBudget) (*Result, error) { if t == nil || t.Root == nil { return &Result{}, nil } + if res, ok := s.selectOnPersistedPages(ctx, t, query); ok { + return res, nil + } + return s.selectOnSectionTree(ctx, t, query, budget) +} + +// selectOnPersistedPages navigates the persisted table of contents over +// the persisted pages. ok is false when either is missing, so the +// caller falls back; an error from navigation itself is returned as a +// Result error through ok=true. +func (s *JudgeWalkStrategy) selectOnPersistedPages(ctx context.Context, t *tree.Tree, query string) (*Result, bool) { + if s.TOC == nil || s.Pages == nil { + return nil, false + } + raw, err := s.TOC.GetTOC(ctx, t.DocumentID) + if err != nil || len(raw) == 0 { + return nil, false + } + var nodes []tree.TOCNode + if err := json.Unmarshal(raw, &nodes); err != nil || len(nodes) == 0 { + return nil, false + } + pages, err := s.Pages.LoadPages(ctx, t.DocumentID) + if err != nil || len(pages) == 0 { + return nil, false + } + byNum := make(map[int]NavPage, len(pages)) + for _, p := range pages { + byNum[p.Number] = p + } + var leaves []NavLeaf + var walk func(ns []tree.TOCNode, path string) + walk = func(ns []tree.TOCNode, path string) { + for _, n := range ns { + p := n.Title + if path != "" { + p = path + " > " + n.Title + } + if len(n.Nodes) > 0 { + walk(n.Nodes, p) + continue + } + if n.StartPage > 0 && n.EndPage >= n.StartPage { + leaves = append(leaves, NavLeaf{ID: n.NodeID, Title: n.Title, Path: p, Start: n.StartPage, End: n.EndPage, Summary: n.Summary}) + } + } + } + walk(nodes, "") + if len(leaves) == 0 { + return nil, false + } + load := func(_ context.Context, leaf NavLeaf) ([]NavPage, error) { + var ps []NavPage + for n := leaf.Start; n <= leaf.End; n++ { + if p, ok := byNum[n]; ok { + ps = append(ps, p) + } + } + return ps, nil + } + nav, err := s.Navigator.Navigate(ctx, query, leaves, load) + if err != nil { + return &Result{ModelUsed: "judge"}, false + } + // The API answers in sections. Each evidence page names the sections + // that cover it; the cited pages are the evidence pages themselves. + sections := flattenSectionsByPage(t) + var ranges []pageRange + conf := map[tree.SectionID]float64{} + var ids []tree.SectionID + seen := map[tree.SectionID]bool{} + for _, ev := range nav.Evidence { + pg := ev.Page.Number + ranges = append(ranges, pageRange{Start: pg, End: pg}) + for _, id := range sectionsOverlapping(sections, []pageRange{{Start: pg, End: pg}}) { + if !seen[id] { + seen[id] = true + ids = append(ids, id) + } + if ev.P > conf[id] { + conf[id] = ev.P + } + } + } + best := 0.0 + if len(nav.Evidence) > 0 { + best = nav.Evidence[0].P + } + return &Result{ + SelectedIDs: ids, + Confidences: conf, + Confidence: best, + CitedPages: rangesToPairs(ranges), + ModelUsed: "judge", + Usage: nav.Usage, + HopsTaken: nav.Requests, + }, true +} + +// selectOnSectionTree is the fallback: the parser's section tree, each +// section's body chunked into page-sized units. +func (s *JudgeWalkStrategy) selectOnSectionTree(ctx context.Context, t *tree.Tree, query string, _ ContextBudget) (*Result, error) { sections := flattenSectionsByPage(t) byID := map[string]sectionPageEntry{} paths := sectionPaths(t) diff --git a/pkg/retrieval/judgewalk_test.go b/pkg/retrieval/judgewalk_test.go index 12552ce..0ef577e 100644 --- a/pkg/retrieval/judgewalk_test.go +++ b/pkg/retrieval/judgewalk_test.go @@ -309,3 +309,61 @@ func TestNavigateFillsThePageBudgetOnAFineTree(t *testing.T) { t.Errorf("full read should be MaxPages=10, got %d", len(res.Pages)) } } + +type fakeTOC struct{ raw []byte } + +func (f fakeTOC) GetTOC(context.Context, tree.DocumentID) ([]byte, error) { return f.raw, nil } + +type fakePages struct{ pages []NavPage } + +func (f fakePages) LoadPages(context.Context, tree.DocumentID) ([]NavPage, error) { + return f.pages, nil +} + +// With a persisted table of contents and pages, navigation runs over +// them — sub-sections, real page ranges — and the API still gets the +// sections that cover the evidence pages, plus the pages themselves. +func TestJudgeWalkUsesThePersistedTOCAndPages(t *testing.T) { + j, _ := navJudge("note 21", "class action") + s := NewJudgeWalkStrategy(j) + s.TOC = fakeTOC{raw: []byte(`[{"node_id":"toc_2","title":"PART II","start_page":21,"end_page":126,"nodes":[ + {"node_id":"toc_2_5","title":"Item 8. Financial Statements","start_page":54,"end_page":125,"nodes":[ + {"node_id":"toc_2_5_1","title":"Note 1 - Policies","start_page":63,"end_page":73}, + {"node_id":"toc_2_5_21","title":"Note 21 - Legal Proceedings","start_page":113,"end_page":114}]}, + {"node_id":"toc_2_6","title":"Item 9. Changes","start_page":126,"end_page":126}]}]`)} + var pages []NavPage + for p := 21; p <= 126; p++ { + text := "prose" + if p == 113 { + text = "Note 21 - Legal Proceedings\nA class action filed in 2019 remains pending." + } + pages = append(pages, NavPage{Number: p, Text: text}) + } + s.Pages = fakePages{pages: pages} + // The section tree the API answers in: Item 8's body is one section + // spanning 54–125 as the parser saw it. + tr := &tree.Tree{DocumentID: "d", Root: &tree.Section{ID: "root", Children: []*tree.Section{ + {ID: "s7", Title: "Item 7", PageStart: 22, PageEnd: 53, ContentRef: "r7"}, + {ID: "s8", Title: "Item 8", PageStart: 54, PageEnd: 125, ContentRef: "r8"}, + }}} + res, err := s.SelectWithCost(context.Background(), tr, "legal proceedings?", ContextBudget{}) + if err != nil { + t.Fatal(err) + } + if len(res.SelectedIDs) != 1 || res.SelectedIDs[0] != "s8" { + t.Errorf("sections covering the evidence page: %v want [s8]", res.SelectedIDs) + } + if len(res.CitedPages) == 0 || res.CitedPages[0] != [2]int{113, 113} { + t.Errorf("cited pages should be the evidence pages: %v", res.CitedPages) + } + if res.ModelUsed != "judge" || res.Usage.LLMCalls == 0 { + t.Errorf("usage/model: %+v", res) + } + // Without a page store it falls back to the section tree and still answers. + s.Pages = nil + s.PageLoader = mapLoader{"r8": "Note 21 - Legal Proceedings\nA class action filed in 2019 remains pending.", "r7": "prose"} + res, err = s.SelectWithCost(context.Background(), tr, "legal proceedings?", ContextBudget{}) + if err != nil || len(res.SelectedIDs) == 0 { + t.Errorf("fallback path: %v %+v", err, res) + } +} From bb8ea8a22bbda375a045bc561983bec30e46f90f Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Sat, 19 Sep 2026 14:19:43 +0100 Subject: [PATCH 07/15] =?UTF-8?q?fix(api):=20page-based=20retrieval=20retu?= =?UTF-8?q?rns=20its=20pages=20=E2=80=94=20evidence=20pages=20lead=20/v1/q?= =?UTF-8?q?uery's=20sections?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HAL-1390. Judgewalk found the right pages (navbench 0.90) and /v1/query then answered in the parser's sections covering them, whose page attribution is what HAL-1375 showed to be unreliable: hit@5 0.475 and the answer in the first result 0 of 40 on FinanceBench. retrieval.Result carries EvidencePages — page number, owning section title, page text, the Judge's confidence — and judgewalk fills it from its evidence set. /v1/query returns them as the leading sections (id page_, page set) ahead of any tree sections, and adds cited_pages. Section-based strategies are unchanged. --- internal/api/abstention_test.go | 22 ++++++++++++++ internal/api/server.go | 54 ++++++++++++++++++++++++++++----- pkg/retrieval/judgewalk.go | 23 +++++++++----- pkg/retrieval/strategy.go | 15 +++++++++ 4 files changed, 100 insertions(+), 14 deletions(-) diff --git a/internal/api/abstention_test.go b/internal/api/abstention_test.go index 48ed4d6..c21a870 100644 --- a/internal/api/abstention_test.go +++ b/internal/api/abstention_test.go @@ -338,3 +338,25 @@ func TestRespondAbstainedAnswerSkipsSynthesis(t *testing.T) { var _ = bytes.NewReader var _ = io.EOF var _ = abstentionRouter + +// A page-based strategy's evidence pages are returned as-is, ahead of +// tree sections, with the page number a client can cite (HAL-1390). +func TestEvidencePageSections(t *testing.T) { + t.Parallel() + got := evidencePageSections([]retrieval.EvidencePage{ + {Page: 113, Title: "Note 21 - Legal Proceedings", Text: "A class action filed in 2019 remains pending.", Confidence: 0.91}, + {Page: 7, Text: "Suppliers"}, + }) + if len(got) != 2 { + t.Fatalf("len %d", len(got)) + } + if got[0]["id"] != "page_113" || got[0]["page"] != 113 || got[0]["confidence"] != 0.91 { + t.Errorf("first: %+v", got[0]) + } + if title, _ := got[0]["title"].(string); !strings.Contains(title, "Note 21") || !strings.HasSuffix(title, "p.113") { + t.Errorf("title %q", title) + } + if got[1]["title"] != "Page 7 · p.7" { + t.Errorf("untitled page: %v", got[1]["title"]) + } +} diff --git a/internal/api/server.go b/internal/api/server.go index 2660d75..a8b6493 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -606,7 +606,7 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { totalUsage := retrieval.Usage{} plan, planUsage := d.runPlanner(r.Context(), body.Query, body.EnablePlanning) totalUsage.Add(planUsage) - ids, confidences, selUsage, err := d.runSelectionWithUsage(r.Context(), t, plan, body.Query, budget) + ids, confidences, selUsage, selResult, err := d.runSelectionResult(r.Context(), t, plan, body.Query, budget) if err != nil { d.Logger.Error("query: strategy failed", "err", err, "document_id", body.DocumentID) writeErr(w, http.StatusInternalServerError, "retrieval failed: "+err.Error()) @@ -675,6 +675,12 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { sections := make([]map[string]any, 0, len(enriched)) finalIDs := make([]tree.SectionID, 0, len(enriched)) + // A page-based strategy's evidence pages lead the list, as-is: the + // page the Judge found holds the answer; a section mapped to it by + // the parser's page attribution may not (HAL-1390). + if selResult != nil { + sections = append(sections, evidencePageSections(selResult.EvidencePages)...) + } for _, e := range enriched { sections = append(sections, sectionWithContentToMap(e)) finalIDs = append(finalIDs, e.sec.ID) @@ -704,6 +710,9 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) { if plan != nil { resp["plan"] = plan } + if selResult != nil && len(selResult.CitedPages) > 0 { + resp["cited_pages"] = selResult.CitedPages + } // Surface the confidence map on the response when present. Only the // finalIDs survive truncation, so trim accordingly. Empty map → // omit so the field stays absent when no signal was available. @@ -1400,24 +1409,33 @@ func (d Deps) runSelectionWithUsage(ctx context.Context, t *tree.Tree, plan *ret // plan is multi-hop AND decomposition is enabled, and surfaces // confidences for the Phase 2.4 abstention check. func (d Deps) runSelectionFull(ctx context.Context, t *tree.Tree, plan *retrieval.Plan, query string, budget retrieval.ContextBudget) ([]tree.SectionID, map[tree.SectionID]float64, retrieval.Usage, error) { + ids, conf, usage, _, err := d.runSelectionResult(ctx, t, plan, query, budget) + return ids, conf, usage, err +} + +// runSelectionResult is runSelectionFull plus the strategy's Result, +// for callers that need what only a page-based strategy carries: +// evidence pages and cited pages. nil when the path had no Result. +func (d Deps) runSelectionResult(ctx context.Context, t *tree.Tree, plan *retrieval.Plan, query string, budget retrieval.ContextBudget) ([]tree.SectionID, map[tree.SectionID]float64, retrieval.Usage, *retrieval.Result, error) { if d.shouldDecompose(plan) { - return retrieval.NewDecomposer(d.Strategy).DecomposedSelectWithConfidences(ctx, t, plan, query, budget) + ids, conf, usage, err := retrieval.NewDecomposer(d.Strategy).DecomposedSelectWithConfidences(ctx, t, plan, query, budget) + return ids, conf, usage, nil, err } if cs, ok := d.Strategy.(retrieval.CostStrategy); ok { res, err := cs.SelectWithCost(ctx, t, query, budget) if err != nil { - return nil, nil, retrieval.Usage{}, err + return nil, nil, retrieval.Usage{}, nil, err } if res == nil { - return nil, nil, retrieval.Usage{}, nil + return nil, nil, retrieval.Usage{}, nil, nil } - return res.SelectedIDs, res.Confidences, res.Usage, nil + return res.SelectedIDs, res.Confidences, res.Usage, res, nil } ids, err := d.Strategy.Select(ctx, t, query, budget) if err != nil { - return nil, nil, retrieval.Usage{}, err + return nil, nil, retrieval.Usage{}, nil, err } - return ids, nil, retrieval.Usage{}, nil + return ids, nil, retrieval.Usage{}, nil, nil } // shouldDecompose returns true when the plan is multi-hop AND @@ -1687,6 +1705,28 @@ func (d Deps) respondAbstained(w http.ResponseWriter, docID tree.DocumentID, que writeJSON(w, http.StatusOK, resp) } +// evidencePageSections renders a page-based strategy's evidence pages +// in the sections shape: id "page_", the owning section's title with +// the page number, the page's text, and the Judge's confidence. +func evidencePageSections(pages []retrieval.EvidencePage) []map[string]any { + out := make([]map[string]any, 0, len(pages)) + for _, ep := range pages { + title := ep.Title + if title == "" { + title = fmt.Sprintf("Page %d", ep.Page) + } + out = append(out, map[string]any{ + "id": fmt.Sprintf("page_%d", ep.Page), + "title": fmt.Sprintf("%s · p.%d", title, ep.Page), + "content": ep.Text, + "page": ep.Page, + "confidence": ep.Confidence, + "token_count": len(ep.Text) / 4, + }) + } + return out +} + // usageJSON is the wire shape of retrieval.Usage on /v1/query, the same // keys /v1/answer uses for total_usage. func usageJSON(u retrieval.Usage) map[string]any { diff --git a/pkg/retrieval/judgewalk.go b/pkg/retrieval/judgewalk.go index 64a19f0..5d38039 100644 --- a/pkg/retrieval/judgewalk.go +++ b/pkg/retrieval/judgewalk.go @@ -691,14 +691,23 @@ func (s *JudgeWalkStrategy) selectOnPersistedPages(ctx context.Context, t *tree. if len(nav.Evidence) > 0 { best = nav.Evidence[0].P } + leafTitle := map[string]string{} + for _, l := range leaves { + leafTitle[l.ID] = l.Title + } + evidence := make([]EvidencePage, 0, len(nav.Evidence)) + for _, ev := range nav.Evidence { + evidence = append(evidence, EvidencePage{Page: ev.Page.Number, Title: leafTitle[ev.Page.LeafID], Text: ev.Page.Text, Confidence: ev.P}) + } return &Result{ - SelectedIDs: ids, - Confidences: conf, - Confidence: best, - CitedPages: rangesToPairs(ranges), - ModelUsed: "judge", - Usage: nav.Usage, - HopsTaken: nav.Requests, + SelectedIDs: ids, + Confidences: conf, + Confidence: best, + CitedPages: rangesToPairs(ranges), + EvidencePages: evidence, + ModelUsed: "judge", + Usage: nav.Usage, + HopsTaken: nav.Requests, }, true } diff --git a/pkg/retrieval/strategy.go b/pkg/retrieval/strategy.go index 2ddd526..6647a91 100644 --- a/pkg/retrieval/strategy.go +++ b/pkg/retrieval/strategy.go @@ -98,6 +98,13 @@ type Result struct { // 102-104 leaves a concrete page footprint behind. PagesRead []PageReadEntry `json:"pages_read,omitempty"` + // EvidencePages are the pages a page-based strategy judged to hold + // the answer, with their text — the unit such a strategy actually + // found, returned as-is rather than mapped back to sections whose + // page attribution may be wrong (HAL-1390). Empty for section-based + // strategies. + EvidencePages []EvidencePage `json:"evidence_pages,omitempty"` + // CitedPages is the FINAL set of page ranges the answer commits // to — the model's cited_pages after dedup and the confidence cap, // NOT every page it read (that is PagesRead). Page-based strategies @@ -117,6 +124,14 @@ type Result struct { Confidence float64 `json:"confidence,omitempty"` } +// EvidencePage is one page a page-based strategy returns as evidence. +type EvidencePage struct { + Page int `json:"page"` + Title string `json:"title,omitempty"` // the section the page belongs to + Text string `json:"text"` + Confidence float64 `json:"confidence"` +} + // PageReadEntry is one get_pages tool call that materialised during a // page-based retrieval loop. StartPage and EndPage are inclusive, // 1-indexed. SectionIDs lists every section whose [PageStart,PageEnd] From 0e51324e0b309fa390deceb0581a3c48d9adde3e Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Sat, 19 Sep 2026 15:34:52 +0100 Subject: [PATCH 08/15] =?UTF-8?q?docs:=20head-to-head=20against=20chunk-an?= =?UTF-8?q?d-embed=20=E2=80=94=20partial=20final=20row,=20the=20two=20API?= =?UTF-8?q?=20lessons,=20the=20baselines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...2026-09-19-head-to-head-chunk-and-embed.md | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 docs/evaluations/2026-09-19-head-to-head-chunk-and-embed.md diff --git a/docs/evaluations/2026-09-19-head-to-head-chunk-and-embed.md b/docs/evaluations/2026-09-19-head-to-head-chunk-and-embed.md new file mode 100644 index 0000000..dfdeb96 --- /dev/null +++ b/docs/evaluations/2026-09-19-head-to-head-chunk-and-embed.md @@ -0,0 +1,89 @@ +# Vectorless on Jev against chunk-and-embed, on FinanceBench + +**Date:** 2026-09-19 +**Harness:** [vectorless-bench](https://github.com/hallelx2/vectorless-bench) `configs/financebench_jev.yaml` — the engine reached through the Python SDK, the baselines in-process +**Corpus:** FinanceBench, 19 filings with questions among the 21 downloaded (68–549 pages), 40 questions, k=5, two repeats +**Engine:** `cmd/engine --local`, `ingest.mode: toc`, `retrieval.strategy: judgewalk`, abstention off, caches off, Judge on, no generative call anywhere in retrieval +**Question:** the post claims chunk-and-embed cannot be this fast and this exact. Is that measured? + +## Result + +The 30 completed questions of the final run are the ones that ran before +the TypeSafe account's credits were exhausted (402 at 15:07). For every +one of them, each gold evidence page is among the returned page units +and the gold page is the first unit returned. The four questions +`navbench` missed (three Boeing, one Pfizer) may sit among the ten not +yet run, so the full-run number is expected between 0.90 and 1.00, not +1.00. The row will be replaced when the run completes. + +| system | how it retrieves | F1@5 | hit@5 | answer span in top-1 | p50 / query | $ / query | ingest, 19 filings | deterministic across repeats | +|---|---|---|---|---|---|---|---|---| +| **Vectorless, judgewalk on persisted pages, pages returned** (PR #68) — **partial: first 30 of 40, one repeat**; the Judge's credits ran out at 15:07 | Jev ranks the TOC's sections, then page heads, then pages; the evidence pages are returned as-is, ahead of any section | 0.631 | **1.000** (30/30) | **1.000** (30/30) | 36 s | $0.0037 | 19 filings ≈ 48 min with sub-section splitting | _repeat 2 not run_ | +| Vectorless, judgewalk on persisted pages, sections mapped by page range (PR #68 before HAL-1390) | same navigation; the parser's sections covering the evidence pages returned | 0.197 | 0.475 | 0.000 | 29 s | $0.0040 | — | — | +| Vectorless, judgewalk on the section tree (first pass) | same navigation over the parser's sections and their bodies | 0.453 | 0.650 | 0.575 | 50 s | $0.0067 | 1,080 s (57 s / filing) | 0.43 exact, 0.69 Jaccard | +| chunk-and-embed, BGE-small | 512-token chunks, bge-small-en-v1.5 on the CPU, cosine top-5 | 0.170 | 0.375 | 0.225 | 49 ms | $0 | 1,972 s (104 s / filing, 4 threads) | 1.00 | +| BM25 | 512-token chunks, BM25 top-5 | 0.096 | 0.200 | 0.075 | 48 ms | $0 | 2 s | 1.00 | + +Scoring: a question's gold is FinanceBench's evidence text and answer; a +returned unit is a hit when it contains the gold span (numbers matched +as numbers). F1@5 is the harness's primary quality; "answer span in +top-1" is whether the first returned unit contains it. + +## What the runs taught, in order + +The server's judgewalk was navigating the **parser's section tree** over +section bodies. The Jev-built table of contents was persisted for +treewalk alone, and per-page text was never persisted. That scored +hit@5 0.65 where the same navigation over real pages had scored 0.90 +in `navbench` — the parser's page attribution is exactly what HAL-1375 +had shown to be unreliable. Ingest now persists the pages beside +`documents.toc_tree`, and judgewalk navigates the persisted TOC over +them (PR #68). + +That alone made things **worse**: hit@5 0.475, the answer in the first +result never. The right pages were found and then mapped back to the +parser's sections covering them by page range — the same unreliable +attribution, one step later. Page-based retrieval now returns its +pages (HAL-1390): `/v1/query` leads with the evidence pages as units +of their own, `page` and `confidence` set. On the 30 questions that +completed, every gold page is returned and it is the first unit. + +Three more things the run surfaced, each fixed on the way: + +- `/v1/query` dropped retrieval's usage, so a client benchmarking + retrieval alone saw $0 (PR #67). +- `ingest.mode: minimal` skips the TOC stage entirely; a `toc` mode now + runs the page-based pipeline and nothing else (PR #67). +- The SDK rejected an abstained response for lacking a model name + (vectorless-sdk #3). Abstention is off for this run so judgewalk's + low-confidence best guess is scored rather than blanked. + +## What the numbers say, and what they do not + +- **Latency.** Chunk-and-embed answers a query in 50 ms; Vectorless on + Jev takes tens of seconds, all of it Judge round-trips. The post's + speed claim is about *ingest* and about the *generative* path it + replaced, not about beating a cosine lookup — say so. +- **Ingest.** Embedding 19 filings with a small model on four CPU + threads took 33 minutes; the Jev TOC stage took 18 minutes for the + same filings with sub-section splitting on, for $0.11. With a GPU the + embedding number collapses; with more Jev concurrency so does ours. +- **Exactness.** Even on the section tree, Vectorless put the answer's + text in its first result 2.6× as often as BGE-small and 7.7× as often + as BM25. That is the citation claim, and it is measured. +- **Determinism.** The baselines are exact across repeats; judgewalk is + not (0.43 exact-match of returned sets), because Jev's probabilities + near the threshold move between calls. A product wants this number + reported, not hidden. +- **Not measured here:** answer quality from the retrieved context + (HAL-1387), the tree-walk chat loop on the same questions, Gemini + embeddings (free-tier quota), and anything on a GPU. + +## Reproduce + +```bash +# engine, local mode, Judge on, page-based pipeline only +VLE_INGEST_MODE=toc TYPESAFE_API_KEY=… ./engine --local -config config.yaml # retrieval.strategy: judgewalk, abstain off, cache off +# bench +cd vectorless-bench && VECTORLESS_BASE_URL=http://localhost:7654 vlbench run --config configs/financebench_jev.yaml +``` From 76546bf529cd39f801570ddc12ac71e74cbe35e1 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Mon, 21 Sep 2026 20:05:11 +0100 Subject: [PATCH 09/15] =?UTF-8?q?docs:=20head-to-head=20final=20row=20?= =?UTF-8?q?=E2=80=94=2040/40=20completed=20on=20the=20page-returning=20eng?= =?UTF-8?q?ine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...2026-09-19-head-to-head-chunk-and-embed.md | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/evaluations/2026-09-19-head-to-head-chunk-and-embed.md b/docs/evaluations/2026-09-19-head-to-head-chunk-and-embed.md index dfdeb96..1ebf425 100644 --- a/docs/evaluations/2026-09-19-head-to-head-chunk-and-embed.md +++ b/docs/evaluations/2026-09-19-head-to-head-chunk-and-embed.md @@ -8,17 +8,19 @@ ## Result -The 30 completed questions of the final run are the ones that ran before -the TypeSafe account's credits were exhausted (402 at 15:07). For every -one of them, each gold evidence page is among the returned page units -and the gold page is the first unit returned. The four questions -`navbench` missed (three Boeing, one Pfizer) may sit among the ten not -yet run, so the full-run number is expected between 0.90 and 1.00, not -1.00. The row will be replaced when the run completes. +The final run completed all 40 questions and both repeats with no +errors and no engine restart. Every gold evidence page is among the +returned pages for 34 of 40 questions; hit@5 (any returned unit holds +the answer text) is 37 of 40; the answer text is in the first returned +unit for 30 of 40. The six page misses are the four `navbench` had +already found (three Boeing, one Pfizer) plus two where a gold page was +one page off the returned one (Pfizer 70–71 → 71; Verizon 23 and 56 → +23 and 57) — a page-boundary question for the full-page pass, not a +navigation one. | system | how it retrieves | F1@5 | hit@5 | answer span in top-1 | p50 / query | $ / query | ingest, 19 filings | deterministic across repeats | |---|---|---|---|---|---|---|---|---| -| **Vectorless, judgewalk on persisted pages, pages returned** (PR #68) — **partial: first 30 of 40, one repeat**; the Judge's credits ran out at 15:07 | Jev ranks the TOC's sections, then page heads, then pages; the evidence pages are returned as-is, ahead of any section | 0.631 | **1.000** (30/30) | **1.000** (30/30) | 36 s | $0.0037 | 19 filings ≈ 48 min with sub-section splitting | _repeat 2 not run_ | +| **Vectorless, judgewalk on persisted pages, pages returned** (PR #68, final run 2026-09-21) | Jev ranks the TOC's sections, then page heads, then pages; the evidence pages are returned as-is, ahead of any section | 0.498 | **0.925** | **0.750** | 37 s (p95 75 s) | $0.0040 | 1,111 s (58 s / filing, sub-section splitting on) | 0.38 exact, 0.83 Jaccard | | Vectorless, judgewalk on persisted pages, sections mapped by page range (PR #68 before HAL-1390) | same navigation; the parser's sections covering the evidence pages returned | 0.197 | 0.475 | 0.000 | 29 s | $0.0040 | — | — | | Vectorless, judgewalk on the section tree (first pass) | same navigation over the parser's sections and their bodies | 0.453 | 0.650 | 0.575 | 50 s | $0.0067 | 1,080 s (57 s / filing) | 0.43 exact, 0.69 Jaccard | | chunk-and-embed, BGE-small | 512-token chunks, bge-small-en-v1.5 on the CPU, cosine top-5 | 0.170 | 0.375 | 0.225 | 49 ms | $0 | 1,972 s (104 s / filing, 4 threads) | 1.00 | @@ -45,8 +47,8 @@ result never. The right pages were found and then mapped back to the parser's sections covering them by page range — the same unreliable attribution, one step later. Page-based retrieval now returns its pages (HAL-1390): `/v1/query` leads with the evidence pages as units -of their own, `page` and `confidence` set. On the 30 questions that -completed, every gold page is returned and it is the first unit. +of their own, `page` and `confidence` set. On the full 40, hit@5 went +0.475 → 0.925 and the answer in the first unit 0.000 → 0.750. Three more things the run surfaced, each fixed on the way: From a0679b7857c4023cc0bf0963c991268ecd867812 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Fri, 25 Sep 2026 00:39:56 +0100 Subject: [PATCH 10/15] feat(ingest): the splitter recurses into the sub-leaves it creates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 70-page Item 8 split into its notes and stopped. A 28-page Note 1 with its own headings stayed one leaf, because the splitter ran once over the leaves the contents pass produced and never looked at the children it made. It now descends into each generation until no leaf exceeds the threshold or splitMaxDepth (4) is reached — the same two sources, a nested index when there is one and heading-shaped lines when there is not, and the same per-leaf Judge confirmation. A 10-K reaches level 3 (part > item > note); the cap leaves room for a note's own headings without recursing into paragraphs. Co-Authored-By: Claude Opus 5 --- pkg/ingest/toc_split.go | 27 +++++++++--- pkg/ingest/toc_split_test.go | 83 ++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 7 deletions(-) diff --git a/pkg/ingest/toc_split.go b/pkg/ingest/toc_split.go index 6615215..a86745c 100644 --- a/pkg/ingest/toc_split.go +++ b/pkg/ingest/toc_split.go @@ -57,10 +57,17 @@ var ( reSplitNumbered = regexp.MustCompile(`(?i)^(note|item|section|part)\s+\d+[a-c]?\b`) ) +// splitMaxDepth bounds how far the splitter descends into the +// sub-leaves it creates. A 10-K reaches level 3 (part > item > +// note); 4 leaves room for a note's own sub-headings without letting +// a pathological document recurse into paragraphs. +const splitMaxDepth = 4 + // splitLargeLeaves walks the tree and splits every leaf whose span -// exceeds over pages. It runs after end pages are derived, so spans are -// known, and re-derives them for the children it adds. Returns how many -// sub-leaves were added. +// exceeds over pages, then splits the sub-leaves it made, until no +// leaf is over the threshold or splitMaxDepth is reached. Runs after +// end pages are derived, so spans are known, and re-derives them for +// each generation of children. Returns how many sub-leaves were added. func (b *TOCBuilder) splitLargeLeaves(ctx context.Context, nodes []tree.TOCNode, pages []PageText, over int, usage *Usage) int { if over <= 0 || b.Judge == nil { return 0 @@ -70,12 +77,15 @@ func (b *TOCBuilder) splitLargeLeaves(ctx context.Context, nodes []tree.TOCNode, byPage[p.PageNumber] = p.Text } added := 0 - var walk func(ns []tree.TOCNode) - walk = func(ns []tree.TOCNode) { + var walk func(ns []tree.TOCNode, depth int) + walk = func(ns []tree.TOCNode, depth int) { for i := range ns { n := &ns[i] if len(n.Nodes) > 0 { - walk(n.Nodes) + walk(n.Nodes, depth+1) + continue + } + if depth >= splitMaxDepth { continue } if n.StartPage <= 0 || n.EndPage < n.StartPage || n.EndPage-n.StartPage+1 <= over { @@ -103,9 +113,12 @@ func (b *TOCBuilder) splitLargeLeaves(ctx context.Context, nodes []tree.TOCNode, deriveEndPagesIn(subs, n.EndPage) n.Nodes = subs added += len(subs) + // Descend into what was just made: a 70-page Item 8 splits + // into notes, and a 28-page note has its own headings. + walk(n.Nodes, depth+1) } } - walk(nodes) + walk(nodes, 1) return added } diff --git a/pkg/ingest/toc_split_test.go b/pkg/ingest/toc_split_test.go index b041e37..69b8ca5 100644 --- a/pkg/ingest/toc_split_test.go +++ b/pkg/ingest/toc_split_test.go @@ -2,6 +2,7 @@ package ingest import ( "context" + "fmt" "strings" "testing" @@ -255,3 +256,85 @@ func titlesOfNodes(ns []tree.TOCNode) []string { } return out } + +// A 70-page Item 8 splits into notes, and a note long enough to have +// its own headings splits again — the same two sources, one level down. +func TestSplitRecursesIntoItsOwnSubLeaves(t *testing.T) { + ps := []PageText{ + {54, "Item 8. Financial Statements\nIndex to the Consolidated Financial Statements\nPage\nNote 1 - Summary of Significant Accounting Policies 60\nNote 2 - Goodwill 95"}, + } + for p := 55; p <= 100; p++ { + text := "Table of Contents\nprose about accounting " + strings.Repeat("x ", 40) + switch p { + case 60: + text = "Note 1 - Summary of Significant Accounting Policies\nPrinciples of Consolidation\nThe consolidated statements include." + case 72: + text = "Table of Contents\nRevenue and Related Cost Recognition\nWe recognize revenue when control transfers." + case 84: + text = "Table of Contents\nUse of Estimates\nEstimates are used throughout." + case 95: + text = "Note 2 - Goodwill\nGoodwill is tested annually." + } + ps = append(ps, PageText{p, text}) + } + b := &TOCBuilder{Judge: splitJudge("note", "principles of consolidation", "revenue and related", "use of estimates")} + nodes := []tree.TOCNode{{Structure: "2.5", Title: "Item 8. Financial Statements", StartPage: 54, EndPage: 100}} + var usage Usage + if n := b.splitLargeLeaves(context.Background(), nodes, ps, 20, &usage); n < 4 { + t.Fatalf("sub-leaves added: %d", n) + } + item8 := nodes[0] + if len(item8.Nodes) < 2 { + t.Fatalf("Item 8 children: %v", titlesOfNodes(item8.Nodes)) + } + var note1 *tree.TOCNode + for i := range item8.Nodes { + if strings.HasPrefix(item8.Nodes[i].Title, "Note 1") { + note1 = &item8.Nodes[i] + } + } + if note1 == nil { + t.Fatalf("Note 1 missing: %v", titlesOfNodes(item8.Nodes)) + } + // Note 1 spans 60–94, over the threshold, so it split again. + if len(note1.Nodes) < 2 { + t.Errorf("Note 1 (%d-%d) should have split at its own headings, children: %v", + note1.StartPage, note1.EndPage, titlesOfNodes(note1.Nodes)) + } + for _, c := range note1.Nodes { + if !strings.HasPrefix(c.Structure, note1.Structure+".") { + t.Errorf("grandchild structure %q does not nest under %q", c.Structure, note1.Structure) + } + if c.StartPage < note1.StartPage || c.EndPage > note1.EndPage { + t.Errorf("grandchild %q spans %d-%d, outside its parent %d-%d", c.Title, c.StartPage, c.EndPage, note1.StartPage, note1.EndPage) + } + } +} + +// The depth cap stops the descent even when leaves stay oversized. +func TestSplitStopsAtMaxDepth(t *testing.T) { + var ps []PageText + for p := 1; p <= 120; p++ { + text := "Section Heading " + fmt.Sprint(p%7) + "\nprose " + strings.Repeat("y ", 40) + ps = append(ps, PageText{p, text}) + } + b := &TOCBuilder{Judge: splitJudge("section heading")} + nodes := []tree.TOCNode{{Structure: "1", Title: "Everything", StartPage: 1, EndPage: 120}} + var usage Usage + b.splitLargeLeaves(context.Background(), nodes, ps, 20, &usage) + var deepest func(ns []tree.TOCNode, d int) int + deepest = func(ns []tree.TOCNode, d int) int { + max := d + for _, n := range ns { + if len(n.Nodes) > 0 { + if x := deepest(n.Nodes, d+1); x > max { + max = x + } + } + } + return max + } + if got := deepest(nodes, 1); got > splitMaxDepth { + t.Errorf("tree reached depth %d, cap is %d", got, splitMaxDepth) + } +} From 302782452072ad01944eadb78dfde66ef83c9f07 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Fri, 25 Sep 2026 01:24:19 +0100 Subject: [PATCH 11/15] =?UTF-8?q?fix(ingest):=20one=20generation=20of=20sp?= =?UTF-8?q?litting=20by=20default=20=E2=80=94=20recursion=20measured=20as?= =?UTF-8?q?=20pure=20cost?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Descending into the sub-leaves the splitter creates bought nothing on FinanceBench and cost plenty. Leaves per filing 69 → 72, median span 1 page either way, the leaf holding a gold page 5 → 6 pages, coverage 47/47 both, navigation's right-section rate 40/40 both and its evidence rate 36/40 → 34/40, pages read per question unchanged at ~41. Ingest paid 278 → 470 Judge requests, $0.13 → $0.21, and 489 → 1,174 seconds. A second generation only reaches leaves the first pass could not find headings in, and those are exactly the ones a second look does not help. SplitGenerations keeps the capability for a document unlike a 10-K; it is off until a corpus shows it earning its cost. Co-Authored-By: Claude Opus 5 --- pkg/ingest/toc_builder.go | 6 ++++ pkg/ingest/toc_split.go | 56 +++++++++++++++++++++++++++--------- pkg/ingest/toc_split_test.go | 36 +++++++++++++++++++++-- 3 files changed, 83 insertions(+), 15 deletions(-) diff --git a/pkg/ingest/toc_builder.go b/pkg/ingest/toc_builder.go index bf6784a..73d716c 100644 --- a/pkg/ingest/toc_builder.go +++ b/pkg/ingest/toc_builder.go @@ -111,6 +111,12 @@ type TOCBuilder struct { // negative disables splitting. SplitLeavesOver int + // SplitGenerations is how many times a path may be split: 1 (the + // default) splits the leaves the contents pass produced and stops. + // More descends into the sub-leaves it creates, which measured as + // pure cost on FinanceBench — see defaultSplitGenerations. + SplitGenerations int + // DetectChars caps the characters of each page sent to detection when // MinimalContext is on. Zero means detectCharsMinimal. A contents page // declares itself in its first couple of thousand characters; the diff --git a/pkg/ingest/toc_split.go b/pkg/ingest/toc_split.go index a86745c..6f45df5 100644 --- a/pkg/ingest/toc_split.go +++ b/pkg/ingest/toc_split.go @@ -57,11 +57,29 @@ var ( reSplitNumbered = regexp.MustCompile(`(?i)^(note|item|section|part)\s+\d+[a-c]?\b`) ) -// splitMaxDepth bounds how far the splitter descends into the -// sub-leaves it creates. A 10-K reaches level 3 (part > item > -// note); 4 leaves room for a note's own sub-headings without letting -// a pathological document recurse into paragraphs. -const splitMaxDepth = 4 +// defaultSplitGenerations is how many times the splitter may split +// along one path. One means a single pass: split the leaves the +// contents pass produced, and stop. +// +// Measured on FinanceBench (2026-09-25): descending into the sub-leaves +// bought nothing and cost plenty. Leaves per filing 69 → 72, median +// span 1 page either way, the leaf holding a gold page 5 → 6 pages, +// coverage 47/47 both, navigation's right-section rate 40/40 both and +// its evidence rate 36/40 → 34/40, with pages read per question +// unchanged at ~41. Ingest paid 278 → 470 Judge requests, $0.13 → +// $0.21, and 489 → 1,174 seconds of wall clock. A second generation +// only reaches leaves the first pass could not find headings in, and +// those are exactly the ones a second look does not help. +// +// The capability stays because a document unlike a 10-K — a deep +// standard, a long contract — may need it; it is off until a corpus +// shows it earning its cost. +const defaultSplitGenerations = 1 + +// splitMaxDepth caps the tree depth the splitter may reach regardless +// of generations, so a pathological document cannot recurse into +// paragraphs. +const splitMaxDepth = 6 // splitLargeLeaves walks the tree and splits every leaf whose span // exceeds over pages, then splits the sub-leaves it made, until no @@ -76,16 +94,17 @@ func (b *TOCBuilder) splitLargeLeaves(ctx context.Context, nodes []tree.TOCNode, for _, p := range pages { byPage[p.PageNumber] = p.Text } + maxGen := b.splitGenerations() added := 0 - var walk func(ns []tree.TOCNode, depth int) - walk = func(ns []tree.TOCNode, depth int) { + var walk func(ns []tree.TOCNode, depth, gen int) + walk = func(ns []tree.TOCNode, depth, gen int) { for i := range ns { n := &ns[i] if len(n.Nodes) > 0 { - walk(n.Nodes, depth+1) + walk(n.Nodes, depth+1, gen) continue } - if depth >= splitMaxDepth { + if depth >= splitMaxDepth || gen >= maxGen { continue } if n.StartPage <= 0 || n.EndPage < n.StartPage || n.EndPage-n.StartPage+1 <= over { @@ -113,15 +132,26 @@ func (b *TOCBuilder) splitLargeLeaves(ctx context.Context, nodes []tree.TOCNode, deriveEndPagesIn(subs, n.EndPage) n.Nodes = subs added += len(subs) - // Descend into what was just made: a 70-page Item 8 splits - // into notes, and a 28-page note has its own headings. - walk(n.Nodes, depth+1) + // Descend into what was just made, when more generations are + // allowed: a 70-page Item 8 splits into notes, and a 28-page + // note has its own headings. + walk(n.Nodes, depth+1, gen+1) } } - walk(nodes, 1) + walk(nodes, 1, 0) return added } +// splitGenerations resolves SplitGenerations: zero selects the default +// of one pass, negative is treated as one, and any larger value allows +// that many generations of splitting along a path. +func (b *TOCBuilder) splitGenerations() int { + if b.SplitGenerations > 0 { + return b.SplitGenerations + } + return defaultSplitGenerations +} + // splitLeaf returns the sub-leaves of one leaf, with start pages, in // page order. func (b *TOCBuilder) splitLeaf(ctx context.Context, leaf *tree.TOCNode, pages []PageText, byPage map[int]string, over int, usage *Usage) ([]tree.TOCNode, error) { diff --git a/pkg/ingest/toc_split_test.go b/pkg/ingest/toc_split_test.go index 69b8ca5..bc37a75 100644 --- a/pkg/ingest/toc_split_test.go +++ b/pkg/ingest/toc_split_test.go @@ -277,7 +277,7 @@ func TestSplitRecursesIntoItsOwnSubLeaves(t *testing.T) { } ps = append(ps, PageText{p, text}) } - b := &TOCBuilder{Judge: splitJudge("note", "principles of consolidation", "revenue and related", "use of estimates")} + b := &TOCBuilder{Judge: splitJudge("note", "principles of consolidation", "revenue and related", "use of estimates"), SplitGenerations: 3} nodes := []tree.TOCNode{{Structure: "2.5", Title: "Item 8. Financial Statements", StartPage: 54, EndPage: 100}} var usage Usage if n := b.splitLargeLeaves(context.Background(), nodes, ps, 20, &usage); n < 4 { @@ -318,7 +318,7 @@ func TestSplitStopsAtMaxDepth(t *testing.T) { text := "Section Heading " + fmt.Sprint(p%7) + "\nprose " + strings.Repeat("y ", 40) ps = append(ps, PageText{p, text}) } - b := &TOCBuilder{Judge: splitJudge("section heading")} + b := &TOCBuilder{Judge: splitJudge("section heading"), SplitGenerations: 99} nodes := []tree.TOCNode{{Structure: "1", Title: "Everything", StartPage: 1, EndPage: 120}} var usage Usage b.splitLargeLeaves(context.Background(), nodes, ps, 20, &usage) @@ -338,3 +338,35 @@ func TestSplitStopsAtMaxDepth(t *testing.T) { t.Errorf("tree reached depth %d, cap is %d", got, splitMaxDepth) } } + +// One generation is the default: the leaves the contents pass produced +// are split, and the sub-leaves are left alone however large they are. +func TestSplitIsOneGenerationByDefault(t *testing.T) { + ps := []PageText{ + {54, "Item 8. Financial Statements\nIndex to the Consolidated Financial Statements\nPage\nNote 1 - Summary of Significant Accounting Policies 60\nNote 2 - Goodwill 95"}, + } + for p := 55; p <= 100; p++ { + text := "Table of Contents\nprose " + strings.Repeat("x ", 40) + switch p { + case 60: + text = "Note 1 - Summary of Significant Accounting Policies\nPrinciples of Consolidation\nThe statements include." + case 72: + text = "Table of Contents\nRevenue and Related Cost Recognition\nWe recognize revenue when control transfers." + case 95: + text = "Note 2 - Goodwill\nGoodwill is tested annually." + } + ps = append(ps, PageText{p, text}) + } + b := &TOCBuilder{Judge: splitJudge("note", "principles of consolidation", "revenue and related")} + nodes := []tree.TOCNode{{Structure: "2.5", Title: "Item 8. Financial Statements", StartPage: 54, EndPage: 100}} + var usage Usage + b.splitLargeLeaves(context.Background(), nodes, ps, 20, &usage) + for _, c := range nodes[0].Nodes { + if len(c.Nodes) > 0 { + t.Errorf("%q split a second time under the default of one generation: %v", c.Title, titlesOfNodes(c.Nodes)) + } + } + if len(nodes[0].Nodes) < 2 { + t.Errorf("the first generation should still split: %v", titlesOfNodes(nodes[0].Nodes)) + } +} From 5f896c835c04f9e121f8773c3ad4ebedb4dd77f5 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Fri, 25 Sep 2026 01:24:39 +0100 Subject: [PATCH 12/15] =?UTF-8?q?docs:=20recursive=20splitting=20measured?= =?UTF-8?q?=20=E2=80=94=20a=20negative=20result?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5 --- .../2026-09-19-leaf-granularity.md | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/evaluations/2026-09-19-leaf-granularity.md b/docs/evaluations/2026-09-19-leaf-granularity.md index cfab008..acf18dc 100644 --- a/docs/evaluations/2026-09-19-leaf-granularity.md +++ b/docs/evaluations/2026-09-19-leaf-granularity.md @@ -62,6 +62,37 @@ Two sources, in order of trust, both "select, don't generate": Operations" to "Contingent Obligations"; its Item 8 became the six statements and twenty-one notes. +## Recursive splitting: a negative result (2026-09-25) + +The splitter ran once over the leaves the contents pass produced, so a +70-page Item 8 split into its notes and a 28-page Note 1 with its own +headings stayed whole. Making it descend through each generation — +same two sources, same per-leaf Judge confirmation, depth-capped — +was built, tested, and measured against the single-pass trees: + +| | one generation | recursive | +|---|---|---| +| leaves per filing, median | 69 | 72 | +| median leaf span | 1 p | 1 p | +| leaf span, p90 | 7 p | 9 p | +| max tree depth | 3 | 4 | +| gold pages inside a leaf | 47 / 47 | 47 / 47 | +| span of the leaf holding a gold page, median | **5 p** | 6 p | +| right section chosen | 40 / 40 | 40 / 40 | +| every gold page in the evidence | **36 / 40** | 34 / 40 | +| pages read / question | 40.6 | 40.8 | +| ingest: Judge requests / cost / wall | **278 / $0.13 / 489 s** | 470 / $0.21 / 1,174 s | + +Nothing improved and ingest paid 1.7× the requests and 2.4× the wall +clock. The reason is visible in the first two rows: after one pass the +median leaf is already a single page, because the per-leaf cap allows +one sub-leaf per two pages. The leaves that remain large are the ones +the splitter could not find headings in — and a second look at the +same text with the same rules does not find them either. + +`SplitGenerations` keeps the capability for a document unlike a 10-K, +defaulting to one. The lesson generalises: recursion is not granularity. + ## What is still missed Four questions, the same four as before the split: Boeing's legal From 4628338b9a9d4ddd6caa47953ce29376691e1396 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Fri, 25 Sep 2026 09:52:01 +0100 Subject: [PATCH 13/15] perf(retrieval): batch page ranking at 6k tokens, not 24k MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured against the provider: latency is flat to ~5k state tokens and then grows faster than the text does (17k tokens → 14.3 s), while concurrency is nearly free (eight parallel 9k requests, four times the work of one 17k request, in half its wall clock). judgewalk packed every request to 24k — a number chosen for the 32k per-question ceiling, never for speed — putting all of them in the penalty region. Also records two negative results: a local embedding cannot pre-narrow the document (BGE-small tops out at 0.875 at k=50, below judgewalk's 0.925) nor the sections the tree already chose (0.875 at k=40 against the Judge's ~0.90+ on the same budget). Co-Authored-By: Claude Opus 5 --- ...ery-latency-and-the-embedding-prefilter.md | 105 ++++++++++++++++++ pkg/retrieval/judgewalk.go | 39 +++++-- 2 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 docs/evaluations/2026-09-25-query-latency-and-the-embedding-prefilter.md diff --git a/docs/evaluations/2026-09-25-query-latency-and-the-embedding-prefilter.md b/docs/evaluations/2026-09-25-query-latency-and-the-embedding-prefilter.md new file mode 100644 index 0000000..3baa106 --- /dev/null +++ b/docs/evaluations/2026-09-25-query-latency-and-the-embedding-prefilter.md @@ -0,0 +1,105 @@ +# Where the 37 seconds goes, and why a cheap pre-filter cannot take it + +**Date:** 2026-09-25 +**Harness:** direct probes against the provider (`scratchpad/latency_shape.py`), offline recall over cached page text (`page_recall.py`, `within_section.py`), [`cmd/navbench`](../../cmd/navbench/main.go) +**Corpus:** FinanceBench, 19 filings with questions (2,936 pages), 40 questions +**Issues:** HAL-1371, HAL-1542 +**Question:** judgewalk answers in 37 s at the median. Retrieval systems people compare us to answer in 49 ms. How much of the gap is recoverable, and by what? + +## Result: most of it is request shape, not work. A local pre-filter cannot help at any scale; batching can. + +## 1. The provider prefers many small requests to few large ones + +One `Noul` per page, real filing pages as state, three repetitions: + +| state tokens | median latency | s per 1k tokens | +|---|---|---| +| 1,589 | 4.2 s | 2.67 | +| 2,724 | 3.6 s | 1.32 | +| 4,919 | 3.8 s | 0.78 | +| 9,253 | 5.9 s | 0.64 | +| 17,159 | 14.3 s | 0.84 | + +Latency is flat to about 5k tokens — fixed per-request overhead — then +grows faster than the text does. And concurrency is close to free: + +| shape | total work | wall clock | +|---|---|---| +| one request, 16 pages | 17k tokens | 8.3 s | +| 4 parallel × 4 pages | 20k tokens | **3.0 s** | +| 8 parallel × 8 pages | 74k tokens | **4.1 s** | + +Four times the total work of a single 17k-token request, in half its +wall clock. + +`judgewalk` was packing each page-ranking request to 24,000 tokens — +a number chosen to respect the provider's 32k per-question ceiling, +never for speed — which put every request in the penalty region. The +budget is now 6k (`defaultNavReqTokens`), which keeps each request in +the flat region and leaves the count to the adaptive limiter. The +batches already fan out concurrently. + +## 2. A local embedding cannot pre-narrow the document + +The appeal is obvious: if BGE-small could shortlist 30 pages in +milliseconds, Jev would judge 30 instead of reading 40, and two round +trips would collapse into one. Measured offline over all 2,936 pages, +"every gold page inside the top k": + +| k | BGE-small | BM25 | +|---|---|---| +| 5 | 0.500 | 0.075 | +| 10 | 0.675 | 0.150 | +| 20 | 0.775 | 0.225 | +| 30 | 0.800 | 0.250 | +| 50 | 0.875 | 0.300 | + +BGE's ceiling at k=50 is 0.875, below the 0.925 judgewalk already +reaches. A pre-filter that caps the pipeline below where it sits is +not a pre-filter, it is a downgrade. + +**Nor inside the sections the tree picked.** Restricting the ranking to +the sections judgewalk chose (median 120 pages, and they contain every +gold page 40/40): + +| k | any gold page in top k | every gold page in top k | +|---|---|---| +| 10 | 0.750 | 0.700 | +| 20 | 0.825 | 0.775 | +| 40 | 0.950 | **0.875** | + +Picking 40 of those 120 pages by embedding similarity scores 0.875; +Jev reading the heads of the same 120 and picking 40 scores ~0.90+. +The Judge's skim beats the embedding at the same budget. + +So the engine's own two-stage narrowing — a structural prior from the +tree, then a read — is better than semantic similarity at every scale +we can measure. That is the thesis, measured from the other side. + +## 3. What is left of the gap + +Three sequential round trips are inherent to the design: rank sections, +skim, read. Everything else is recoverable: + +- **Request shape** (done): the page pass was three ~22k requests at + ~14 s; at 6k it is ~10 requests in the flat region, in flight + together. +- **A needless chain link** (not yet done): the skim only depends on the + ranking because we skim *the chosen sections*. Skim every page instead + and the two requests fire together — one fewer round trip, and more + pages scored, not fewer. +- **Moving the skim to ingest** (HAL-1542): a per-page index built once + makes the skim a lookup over stored answers. + +None of that reaches 49 ms, and it should not be claimed. A system that +answers in 49 ms does no model work at query time, which is exactly the +compression that costs it 0.375 against our 0.925. The honest target is +single-digit seconds with the accuracy intact. + +## Reproduce + +```bash +python scratchpad/latency_shape.py # needs TYPESAFE_API_KEY; ~30 requests, about a cent +python scratchpad/page_recall.py # offline, cached page text, no API calls +python scratchpad/within_section.py # offline +``` diff --git a/pkg/retrieval/judgewalk.go b/pkg/retrieval/judgewalk.go index 5d38039..3e35ce7 100644 --- a/pkg/retrieval/judgewalk.go +++ b/pkg/retrieval/judgewalk.go @@ -112,18 +112,41 @@ type JudgeNavigator struct { // The provider treats the whole state object as shared context for // every question — there is no per-question state — so state plus // the longest question must stay under 32k tokens. Zero selects - // 24k, the same ceiling the TOC stage uses; that is six to eight - // dense filing pages per request. + // defaultNavReqTokens. + // + // This is a LATENCY knob, not just a limit. Batches are sent + // concurrently, so a smaller budget means more requests in flight, + // not more waiting — and the provider is far happier with many small + // requests than a few large ones (see defaultNavReqTokens). RequestBudgetTokens int } const ( - defaultNavThreshold = 0.5 - defaultNavMaxPages = 40 - defaultNavCoarse = 120 - defaultNavHeadChars = 700 - defaultNavPageChars = 6000 - defaultNavReqTokens = 24_000 + defaultNavThreshold = 0.5 + defaultNavMaxPages = 40 + defaultNavCoarse = 120 + defaultNavHeadChars = 700 + defaultNavPageChars = 6000 + // defaultNavReqTokens: measured against the provider on 2026-09-25, + // one Noul per page with real filing pages as state. + // + // state tokens median latency + // 1,589 4.2 s + // 2,724 3.6 s + // 4,919 3.8 s + // 9,253 5.9 s + // 17,159 14.3 s + // + // Latency is flat to about 5k tokens — fixed per-request overhead — + // and then grows faster than the text does. Concurrency is close to + // free: eight parallel 9k-token requests finished in 4.1 s wall, + // four times the total work of a single 17k-token request in half + // its wall clock. + // + // So the right shape is many small requests in flight, not few large + // ones. 6k keeps each request in the flat region; the adaptive + // limiter (llmgate middleware/limit) decides how many run at once. + defaultNavReqTokens = 6_000 navLeafBatch = 120 navMinEvidencePages = 2 navLeafStateMaxChars = 300 From e9a3313fbfba2d681bcac224c76fdfd1985de580 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Fri, 25 Sep 2026 10:05:24 +0100 Subject: [PATCH 14/15] perf(retrieval): stop tokenising to pack batches; revert the request budget to 24k MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two corrections to yesterday's latency work, both from better probes. The batch budget was moved 24k → 6k on a probe that varied pages, which moved state size and question count together and was contaminated by cold start. Holding one variable at a time, warm, shows request shape barely matters: forty pages as ten parallel requests took 2.6 s wall, and as sixteen-page requests 2.6 s each. The one navigation run at 6k came back three times slower. Back to 24k, which no measurement argues against. The real cost was ours. Packing batches tokenised every page with the provider's tokenizer — 4.0 s of CPU for one question's forty pages, before a single request goes out, and the client tokenises the state again per request. Packing now estimates from length, biased to over-estimate; the client's exact check still guards the ceiling. Co-Authored-By: Claude Opus 5 --- pkg/retrieval/judgewalk.go | 55 +++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/pkg/retrieval/judgewalk.go b/pkg/retrieval/judgewalk.go index 3e35ce7..b218e6a 100644 --- a/pkg/retrieval/judgewalk.go +++ b/pkg/retrieval/judgewalk.go @@ -11,7 +11,6 @@ import ( "regexp" "github.com/hallelx2/llmgate" - "github.com/hallelx2/llmgate/judge/typesafe" "github.com/hallelx2/vectorless-engine/pkg/tree" ) @@ -127,26 +126,20 @@ const ( defaultNavCoarse = 120 defaultNavHeadChars = 700 defaultNavPageChars = 6000 - // defaultNavReqTokens: measured against the provider on 2026-09-25, - // one Noul per page with real filing pages as state. + // defaultNavReqTokens bounds one request's state. Measured warm on + // 2026-09-25, one Noul per page: request SHAPE barely matters. + // Forty pages went out as ten parallel requests in 2.6 s wall, and + // as sixteen-page requests in 2.6 s each. An earlier probe appeared + // to punish large requests; that was cold-start contamination — the + // first calls of a session take 5-6 s and the rest settle to + // 1.5-2.6 s. // - // state tokens median latency - // 1,589 4.2 s - // 2,724 3.6 s - // 4,919 3.8 s - // 9,253 5.9 s - // 17,159 14.3 s - // - // Latency is flat to about 5k tokens — fixed per-request overhead — - // and then grows faster than the text does. Concurrency is close to - // free: eight parallel 9k-token requests finished in 4.1 s wall, - // four times the total work of a single 17k-token request in half - // its wall clock. - // - // So the right shape is many small requests in flight, not few large - // ones. 6k keeps each request in the flat region; the adaptive - // limiter (llmgate middleware/limit) decides how many run at once. - defaultNavReqTokens = 6_000 + // This was briefly retuned to 6k on the strength of that bad probe + // and one navigation run came back three times slower (not a paired + // control either). No measured reason to move off 24k, which stays + // under the provider's 32k per-question ceiling with room for the + // question text. + defaultNavReqTokens = 24_000 navLeafBatch = 120 navMinEvidencePages = 2 navLeafStateMaxChars = 300 @@ -521,14 +514,22 @@ func (n *JudgeNavigator) Navigate(ctx context.Context, query string, leaves []Na return out, nil } -// countTokens uses the provider's tokenizer, so the batch budget is -// measured the way the request will be. Dense financial tables run -// near one token per two characters; a bytes/4 guess overflowed. +// countTokens estimates a string's token count for packing batches. +// +// It deliberately does NOT run the provider's tokenizer. Measured +// 2026-09-25: tokenising one question's forty pages costs 4.0 s of +// CPU, and the client tokenises the assembled state again before every +// request — about five seconds per question spent counting rather than +// asking, on a query whose median is thirty-seven. +// +// The packing budget only needs a safe upper bound; the client's own +// check is exact and refuses anything over the ceiling. Dense +// financial tables run near one token per two and a half characters, +// so len/2 over-estimates prose and sits close on tables. +// Over-estimating costs one extra request, under-estimating costs a +// rejected call, so the bias is deliberate. func countTokens(text string) int { - if n, err := typesafe.EstimateTokens(text); err == nil { - return n - } - return len(text)/3 + 1 + return len(text)/2 + 1 } // reReference finds the cross-references a page makes: "see Note 21", From 8bef6d34e79320c465ba635ba73c0bb988a775fd Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Fri, 25 Sep 2026 10:27:33 +0100 Subject: [PATCH 15/15] perf(retrieval): pack batches at the measured token ratio; start the limiter wide Measured end to end on the same 12 FinanceBench questions, accuracy unchanged at 12/12: baseline (tokenised packing, limiter at 4) 4.3 req 37.0 s length packing at len/2 6.6 req 35.1 s length packing at len/4, limiter at 16 4.2 req 28.6 s len/2 over-estimated real filing text by two and a half times (it bills at 4.9 characters per token), halving every batch and spending on extra requests exactly what tokenising had cost. len/4 keeps a fifth of headroom. The limiter started at 4 and a transient failure halved it to 2; AIMD needs twenty consecutive successes to widen by one, which one interactive query never earns. A query's requests are independent, so navbench starts at 16. The evaluation is corrected too: its first section reported that the provider punishes large requests, which was a cold-start artefact of a probe that varied two things at once. Shape barely matters; our own overhead did. Co-Authored-By: Claude Opus 5 --- cmd/navbench/main.go | 7 +- ...ery-latency-and-the-embedding-prefilter.md | 84 +++++++++++-------- pkg/retrieval/judgewalk.go | 16 ++-- 3 files changed, 67 insertions(+), 40 deletions(-) diff --git a/cmd/navbench/main.go b/cmd/navbench/main.go index e8f541c..b07bc12 100644 --- a/cmd/navbench/main.go +++ b/cmd/navbench/main.go @@ -87,7 +87,12 @@ func main() { fmt.Fprintln(os.Stderr, "judge:", err) os.Exit(1) } - lim := limit.New(limit.Config{Initial: 4, OnChange: func(e limit.Event) { + // Initial 16, not 4. A question issues 4-7 requests that do not + // depend on each other, so a limiter that starts at 4 serialises + // them into waves, and AIMD needs twenty successes to widen by one — + // slower than a 40-question run can recover from, especially after a + // transient failure halves it. + lim := limit.New(limit.Config{Initial: 16, OnChange: func(e limit.Event) { fmt.Fprintf(os.Stderr, " limiter %s %d -> %d %v\n", e.Cause, e.From, e.To, e.Err) }}) nav := &retrieval.JudgeNavigator{Judge: retry.NewJudge(retry.Config{MaxRetries: 3})(limit.Judge(lim)(tj)), MaxLeaves: *maxLeaves, MaxPages: *maxPages} diff --git a/docs/evaluations/2026-09-25-query-latency-and-the-embedding-prefilter.md b/docs/evaluations/2026-09-25-query-latency-and-the-embedding-prefilter.md index 3baa106..e6d185d 100644 --- a/docs/evaluations/2026-09-25-query-latency-and-the-embedding-prefilter.md +++ b/docs/evaluations/2026-09-25-query-latency-and-the-embedding-prefilter.md @@ -8,36 +8,55 @@ ## Result: most of it is request shape, not work. A local pre-filter cannot help at any scale; batching can. -## 1. The provider prefers many small requests to few large ones - -One `Noul` per page, real filing pages as state, three repetitions: - -| state tokens | median latency | s per 1k tokens | -|---|---|---| -| 1,589 | 4.2 s | 2.67 | -| 2,724 | 3.6 s | 1.32 | -| 4,919 | 3.8 s | 0.78 | -| 9,253 | 5.9 s | 0.64 | -| 17,159 | 14.3 s | 0.84 | - -Latency is flat to about 5k tokens — fixed per-request overhead — then -grows faster than the text does. And concurrency is close to free: - -| shape | total work | wall clock | -|---|---|---| -| one request, 16 pages | 17k tokens | 8.3 s | -| 4 parallel × 4 pages | 20k tokens | **3.0 s** | -| 8 parallel × 8 pages | 74k tokens | **4.1 s** | - -Four times the total work of a single 17k-token request, in half its -wall clock. - -`judgewalk` was packing each page-ranking request to 24,000 tokens — -a number chosen to respect the provider's 32k per-question ceiling, -never for speed — which put every request in the penalty region. The -budget is now 6k (`defaultNavReqTokens`), which keeps each request in -the flat region and leaves the count to the adaptive limiter. The -batches already fan out concurrently. +## 1. Request shape barely matters. Two of my own bugs did. + +**First probe, and it was wrong.** Varying pages per request moved two +things at once — the text sent and the number of questions asked — and +the session's first calls were cold. It appeared to show large requests +being punished (17k tokens → 14.3 s). Acting on it, the page-ranking +budget was cut 24k → 6k, and a navigation run came back three times +slower. + +**Second probe, one variable at a time, warm.** Holding state near 6k +while varying question count, then holding questions at 4 while varying +text, both curves are flat: everything lands between 1.4 s and 2.6 s. +The first two calls of a session take 5-6 s and then it settles. Forty +pages as ten parallel requests took 2.6 s wall; the same forty as +sixteen-page requests took 2.6 s each. Shape is not the lever. + +**What was the lever, measured end to end on the same 12 questions:** + +| build | requests / question | median s | hit@5 | +|---|---|---|---| +| baseline (24k budget, tokenised packing) | 4.3 | 37.0 | 12/12 | +| length-estimated packing, len/2 | 6.6 | 35.1 | 12/12 | +| **len/4 packing + limiter starting at 16** | **4.2** | **28.6** | 12/12 | + +Two bugs, both ours: + +- **Packing ran the tokenizer.** Deciding which pages go in which + request tokenised all forty pages first — 4.0 s of CPU before a + single call went out, plus the client tokenising the assembled state + again per request. Packing only needs a safe upper bound, so it now + estimates from length. The first attempt used len/2, which + over-estimates real filing text by two and a half times (measured: + 4.9 characters per token), halved every batch, and sent 6.6 requests + where 4.3 had done — cancelling the saving exactly. len/4 keeps a + fifth of headroom and restores the batch size. +- **The limiter was starting at 4 and being halved by transient + failures.** AIMD needs twenty consecutive successes to widen by one, + which a single interactive query never lives long enough to earn; one + failed request left the run at concurrency 2 for most of its length. + A query's four to seven requests are independent, so navbench now + starts at 16 and the run stayed at 16-18 throughout. + +**What is still unexplained.** At 4.2 requests and ~21k tokens each, +warm rates predict roughly 3 s per request and three sequential phases, +so about 9 s. The engine takes 28.6 s. That factor of three is not +accounted for by anything measured here — candidates are provider +behaviour under sustained load versus a short probe, and retry backoff +after transient failures. It should be instrumented per request before +anyone optimises further, rather than guessed at a third time. ## 2. A local embedding cannot pre-narrow the document @@ -81,9 +100,8 @@ we can measure. That is the thesis, measured from the other side. Three sequential round trips are inherent to the design: rank sections, skim, read. Everything else is recoverable: -- **Request shape** (done): the page pass was three ~22k requests at - ~14 s; at 6k it is ~10 requests in the flat region, in flight - together. +- **Our own overhead** (done): tokenised packing and a limiter that + started narrow, together 37 s → 28.6 s on the same questions. - **A needless chain link** (not yet done): the skim only depends on the ranking because we skim *the chosen sections*. Skim every page instead and the two requests fire together — one fewer round trip, and more diff --git a/pkg/retrieval/judgewalk.go b/pkg/retrieval/judgewalk.go index b218e6a..e035ef0 100644 --- a/pkg/retrieval/judgewalk.go +++ b/pkg/retrieval/judgewalk.go @@ -523,13 +523,17 @@ func (n *JudgeNavigator) Navigate(ctx context.Context, query string, leaves []Na // asking, on a query whose median is thirty-seven. // // The packing budget only needs a safe upper bound; the client's own -// check is exact and refuses anything over the ceiling. Dense -// financial tables run near one token per two and a half characters, -// so len/2 over-estimates prose and sits close on tables. -// Over-estimating costs one extra request, under-estimating costs a -// rejected call, so the bias is deliberate. +// check is exact and refuses anything over the ceiling. +// +// Measured on real filing pages: 24,000 characters of page text bill +// as 4,875 tokens, so 4.9 characters per token. len/2 was the first +// guess and over-estimated by two and a half times, halving every +// batch and sending 6.6 requests per question where 4.3 had done — +// which cancelled the CPU saved. len/4 leaves a fifth of headroom +// over the measured ratio, which covers dense numeric tables without +// throwing away batch size. func countTokens(text string) int { - return len(text)/2 + 1 + return len(text)/4 + 1 } // reReference finds the cross-references a page makes: "see Note 21",