feat: persist the tree and file content through an optional store

README.md

@@ -36,6 +36,7 @@ const matches = await docs.search('repair', { pathPrefix: 'repair' });
 | `token`           | `GITHUB_TOKEN` or `GH_TOKEN` | GitHub token                 |
 | `cacheTtlMs.dir`  | `300000`                     | Directory and tree cache TTL |
 | `cacheTtlMs.file` | `600000`                     | File cache TTL               |
+| `store`           | none                         | Persistent cache, see below  |
 
 ### `docs.listDir(path?)`
 
@@ -48,7 +49,12 @@ Returns raw file content.
 
 ### `docs.listAll()`
 
-Lists every Markdown file through GitHub's recursive tree API.
+Lists every Markdown file through GitHub's recursive tree API. Each item carries its Git blob `sha`.
+
+### `docs.peekAll()`
+
+Returns the last known tree without a request: the one fetched in this process, else the one in
+`store`, else `undefined`.
 
 ### `docs.listSections()`
 
@@ -60,6 +66,13 @@ Returns content with its route, section, title, summary, and semantic component
 metadata covers `PageHero`, `FactStrip`, `LinkCard`, `Split`, `TimelineEntry`, and `Figure` without
 imposing a renderer.
 
+### `store`
+
+An object with `read(key): string | undefined` and `write(key, value): void`. The client keeps the
+tree under `tree` and file content under `blob-<sha>`, where `<sha>` is the Git blob id. `getFile`
+serves stored content only when its hash matches the blob id in the last known tree, so a changed
+file is always refetched. Store errors and corrupt entries are ignored; eviction is up to the store.
+
 ### `docs.search(query, options?)`
 
 Searches paths, titles, summaries, Markdown text, and semantic component attributes. Results are

scripts/check-package.mjs

@@ -42,6 +42,7 @@ try {
       'const client = createDocsClient();',
       "if (typeof client.listDir !== 'function') throw new TypeError('invalid client export');",
       "if (typeof client.listAll !== 'function') throw new TypeError('invalid tree export');",
+      "if (client.peekAll() !== undefined) throw new TypeError('invalid peek export');",
       "if (typeof client.listSections !== 'function') throw new TypeError('invalid discovery export');",
       "if (typeof client.getDocument !== 'function') throw new TypeError('invalid document export');",
       "if (typeof client.search !== 'function') throw new TypeError('invalid search export');",
@@ -55,21 +56,24 @@ try {
     join(temporaryDirectory, 'consumer.ts'),
     [
       "import { createDocsClient } from '@nbtca/docs';",
-      "import type { DocComponent, DocItem, DocPage, DocSection, DocsClient, DocsClientOptions, DocsSearchOptions, DocsSearchResult } from '@nbtca/docs';",
-      "const item = { name: 'guide.md', path: 'repair/guide.md', type: 'file' } satisfies DocItem;",
+      "import type { DocComponent, DocItem, DocPage, DocSection, DocsClient, DocsClientOptions, DocsSearchOptions, DocsSearchResult, DocsStore } from '@nbtca/docs';",
+      "const item = { name: 'guide.md', path: 'repair/guide.md', type: 'file', sha: 'b5aaad7d6dda27ea24335cdd4722c8129113f4cd' } satisfies DocItem;",
+      'const store: DocsStore = { read: () => undefined, write: () => undefined };',
       "const component = { name: 'Figure', attributes: { caption: 'Example' } } satisfies DocComponent;",
       "const page = { components: [component], content: '# Guide', name: item.name, path: item.path, route: '/repair/guide', section: 'repair', summary: 'Repair guide', title: 'Guide' } satisfies DocPage;",
       "const section = { count: 1, indexPath: 'repair/index.md', path: 'repair' } satisfies DocSection;",
       "const searchOptions = { pathPrefix: 'repair', limit: 10 } satisfies DocsSearchOptions;",
       "const searchResult = { excerpt: 'Repair guide', name: item.name, path: item.path, route: page.route, score: 42, section: page.section, summary: page.summary, title: page.title } satisfies DocsSearchResult;",
-      "const options = { branch: 'main', cacheTtlMs: { dir: 0, file: 0 } } satisfies DocsClientOptions;",
+      "const options = { branch: 'main', cacheTtlMs: { dir: 0, file: 0 }, store } satisfies DocsClientOptions;",
       'const client: DocsClient = createDocsClient(options);',
       'async function consumePromptContract(docs: DocsClient): Promise<void> {',
       '  const items: DocItem[] = await docs.listAll();',
+      '  const known: DocItem[] | undefined = docs.peekAll();',
       '  const document: DocPage = await docs.getDocument(item.path);',
       "  const results: DocsSearchResult[] = await docs.search('repair', searchOptions);",
       '  docs.clear();',
       '  void items;',
+      '  void known;',
       '  void document;',
       '  void results;',
       '}',

src/__tests__/client.test.ts

@@ -776,6 +776,100 @@ describe('listAll', () => {
   });
 });
 
+describe('persistent store', () => {
+  const guideSha = 'b5aaad7d6dda27ea24335cdd4722c8129113f4cd';
+  const changedSha = 'b125dfa2b24f5e3865648e4e3ca989bf0804bed1';
+  const tree = (sha: string) => ({
+    truncated: false,
+    tree: [{ path: 'repair/guide.md', type: 'blob', sha }],
+  });
+
+  function memoryStore(entries: Record<string, string> = {}) {
+    const map = new Map(Object.entries(entries));
+    return {
+      map,
+      read: (key: string) => map.get(key),
+      write: (key: string, value: string) => {
+        map.set(key, value);
+      },
+    };
+  }
+
+  it('exposes blob shas and persists the tree', async () => {
+    mockFetch({ ok: true, json: async () => tree(guideSha) });
+    const store = memoryStore();
+    const items = await createDocsClient({ store }).listAll();
+    expect(items).toEqual([
+      { name: 'guide.md', path: 'repair/guide.md', type: 'file', sha: guideSha },
+    ]);
+    expect(createDocsClient({ store }).peekAll()).toEqual(items);
+  });
+
+  it('drops shas that are not Git object ids', async () => {
+    mockFetch({ ok: true, json: async () => tree('../escape') });
+    const [item] = await createDocsClient().listAll();
+    expect(item).not.toHaveProperty('sha');
+  });
+
+  it('peeks nothing without a stored or fetched tree', () => {
+    expect(createDocsClient().peekAll()).toBeUndefined();
+    expect(createDocsClient({ store: memoryStore() }).peekAll()).toBeUndefined();
+  });
+
+  it('serves a stored blob matching the known sha without fetching', async () => {
+    const spy = vi.fn().mockRejectedValue(new TypeError('fetch failed'));
+    vi.stubGlobal('fetch', spy);
+    const store = memoryStore({
+      tree: JSON.stringify([
+        { name: 'guide.md', path: 'repair/guide.md', type: 'file', sha: guideSha },
+      ]),
+      [`blob-${guideSha}`]: '# Guide',
+    });
+    await expect(createDocsClient({ store }).getFile('repair/guide.md')).resolves.toBe('# Guide');
+    expect(spy).not.toHaveBeenCalled();
+  });
+
+  it('refetches a changed file and stores it under its own sha', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi
+        .fn()
+        .mockResolvedValueOnce({ ok: true, json: async () => tree(changedSha) })
+        .mockResolvedValueOnce({ ok: true, text: async () => '# Guide v2' }),
+    );
+    const store = memoryStore({ [`blob-${guideSha}`]: '# Guide' });
+    const client = createDocsClient({ store });
+    await client.listAll();
+    await expect(client.getFile('repair/guide.md')).resolves.toBe('# Guide v2');
+    expect(store.map.get(`blob-${changedSha}`)).toBe('# Guide v2');
+  });
+
+  it('ignores corrupt entries and failing stores', async () => {
+    mockFetch({ ok: true, text: async () => '# Guide' });
+    const corrupt = memoryStore({
+      tree: JSON.stringify([
+        { name: 'guide.md', path: 'repair/guide.md', type: 'file', sha: guideSha },
+      ]),
+      [`blob-${guideSha}`]: '# Gui',
+    });
+    await expect(createDocsClient({ store: corrupt }).getFile('repair/guide.md')).resolves.toBe(
+      '# Guide',
+    );
+    expect(createDocsClient({ store: memoryStore({ tree: '{' }) }).peekAll()).toBeUndefined();
+    const failing = {
+      read: () => {
+        throw new Error('EACCES');
+      },
+      write: () => {
+        throw new Error('ENOSPC');
+      },
+    };
+    await expect(createDocsClient({ store: failing }).getFile('repair/guide.md')).resolves.toBe(
+      '# Guide',
+    );
+  });
+});
+
 describe('document discovery', () => {
   it('groups documents by top-level section and identifies section indexes', async () => {
     mockFetch({

src/client.ts

@@ -42,6 +42,8 @@ const SKIP = new Set([
   'docs',
 ]);
 
+const TREE_KEY = 'tree';
+const SHA = /^[0-9a-f]{40}$/;
 const SEARCH_CONCURRENCY = 6;
 const SEARCH_RESULT_LIMIT = 20;
 
@@ -57,6 +59,7 @@ function filterAndSort(raw: GitHubItem[]): DocItem[] {
       name: item.name,
       path: item.path,
       type: item.type === 'dir' ? 'dir' : 'file',
+      ...shaOf(item),
     }))
     .sort((a, b) => {
       if (a.type !== b.type) return a.type === 'dir' ? -1 : 1;
@@ -75,10 +78,45 @@ function filterTree(items: GitHubTreeItem[]): DocItem[] {
       name: item.path.slice(item.path.lastIndexOf('/') + 1),
       path: item.path,
       type: 'file' as const,
+      ...shaOf(item),
     }))
     .sort((a, b) => a.path.localeCompare(b.path));
 }
 
+function shaOf(item: { sha?: unknown }): { sha?: string } {
+  return typeof item.sha === 'string' && SHA.test(item.sha) ? { sha: item.sha } : {};
+}
+
+async function blobSha(content: string): Promise<string> {
+  const body = new TextEncoder().encode(content);
+  const header = new TextEncoder().encode(`blob ${String(body.length)}\0`);
+  const object = new Uint8Array(header.length + body.length);
+  object.set(header);
+  object.set(body, header.length);
+  const digest = new Uint8Array(await crypto.subtle.digest('SHA-1', object));
+  return Array.from(digest, (byte) => byte.toString(16).padStart(2, '0')).join('');
+}
+
+function isDocItem(value: unknown): value is DocItem {
+  return (
+    isRecord(value) &&
+    typeof value.name === 'string' &&
+    typeof value.path === 'string' &&
+    (value.type === 'file' || value.type === 'dir') &&
+    (value.sha === undefined || typeof value.sha === 'string')
+  );
+}
+
+function parseStoredTree(value: string | undefined): DocItem[] | undefined {
+  if (value === undefined) return undefined;
+  try {
+    const items: unknown = JSON.parse(value);
+    return Array.isArray(items) && items.every(isDocItem) ? items : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
 function copyItems(items: DocItem[]): DocItem[] {
   return items.map((item) => ({ ...item }));
 }
@@ -132,11 +170,13 @@ interface GitHubItem {
   name: string;
   path: string;
   type: string;
+  sha?: unknown;
 }
 
 interface GitHubTreeItem {
   path: string;
   type: string;
+  sha?: unknown;
 }
 
 interface GitHubTreeResponse {
@@ -314,8 +354,33 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
   const dirRequests = new Map<string, Promise<DocItem[]>>();
   const fileRequests = new Map<string, Promise<string>>();
   const treeRequests = new Map<string, Promise<DocItem[]>>();
+  const store = options.store;
+  let storedTree: DocItem[] | undefined;
   let cacheGeneration = 0;
 
+  function storeRead(key: string): string | undefined {
+    try {
+      return store?.read(key);
+    } catch {
+      return undefined;
+    }
+  }
+
+  function storeWrite(key: string, value: string): void {
+    try {
+      store?.write(key, value);
+    } catch {
+      return;
+    }
+  }
+
+  function knownTree(): DocItem[] | undefined {
+    const fetched = treeCache.getStale(TREE_KEY);
+    if (fetched) return fetched;
+    storedTree ??= parseStoredTree(storeRead(TREE_KEY));
+    return storedTree;
+  }
+
   function headers(): Record<string, string> {
     const requestHeaders: Record<string, string> = {
       Accept: 'application/vnd.github.v3+json',
@@ -406,7 +471,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
   }
 
   async function loadAll(generation: number): Promise<DocItem[]> {
-    const key = '__tree__';
+    const key = TREE_KEY;
     const url = `${apiRepoUrl}/git/trees/${encodedBranch}?recursive=1`;
     try {
       return await withResponse(url, 20_000, async (response) => {
@@ -426,7 +491,10 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
           );
         }
         const items = filterTree(data.tree);
-        if (generation === cacheGeneration) treeCache.set(key, copyItems(items));
+        if (generation === cacheGeneration) {
+          treeCache.set(key, copyItems(items));
+          if (store) storeWrite(key, JSON.stringify(items));
+        }
         return items;
       });
     } catch (error) {
@@ -436,17 +504,33 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
   }
 
   function listAll(): Promise<DocItem[]> {
-    const key = '__tree__';
-    const hit = treeCache.get(key);
+    const hit = treeCache.get(TREE_KEY);
     if (hit) return Promise.resolve(copyItems(hit));
-    return shareRequest(treeRequests, key, () => loadAll(cacheGeneration)).then(copyItems);
+    return shareRequest(treeRequests, TREE_KEY, () => loadAll(cacheGeneration)).then(copyItems);
+  }
+
+  function peekAll(): DocItem[] | undefined {
+    const items = knownTree();
+    return items && copyItems(items);
   }
 
   async function listSections(): Promise<DocSection[]> {
     return sectionsFromItems(await listAll());
   }
 
+  async function loadStoredFile(path: string): Promise<string | undefined> {
+    const sha = knownTree()?.find((item) => item.path === path)?.sha;
+    if (sha === undefined) return undefined;
+    const content = storeRead(`blob-${sha}`);
+    return content !== undefined && (await blobSha(content)) === sha ? content : undefined;
+  }
+
   async function loadFile(path: string, generation: number): Promise<string> {
+    const stored = store && (await loadStoredFile(path));
+    if (stored !== undefined) {
+      if (generation === cacheGeneration) fileCache.set(path, stored);
+      return stored;
+    }
     const url = `${rawRepoUrl}/${encodedBranch}/${encodePath(path)}`;
     try {
       return await withResponse(url, 15_000, async (response) => {
@@ -457,6 +541,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
         }
         const content = await response.text();
         if (generation === cacheGeneration) fileCache.set(path, content);
+        if (store) storeWrite(`blob-${await blobSha(content)}`, content);
         return content;
       });
     } catch (error) {
@@ -521,6 +606,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
 
   function clear(): void {
     cacheGeneration += 1;
+    storedTree = undefined;
     dirCache.clear();
     fileCache.clear();
     treeCache.clear();
@@ -529,5 +615,5 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
     treeRequests.clear();
   }
 
-  return { listDir, listAll, listSections, getFile, getDocument, search, clear };
+  return { listDir, listAll, peekAll, listSections, getFile, getDocument, search, clear };
 }

src/index.ts

@@ -9,5 +9,6 @@ export type {
   DocsClientOptions,
   DocsSearchOptions,
   DocsSearchResult,
+  DocsStore,
 } from './types.js';
 export { DocsFetchError } from './types.js';