diff --git a/src/hooks/trace-window.test.ts b/src/hooks/trace-window.test.ts new file mode 100644 index 0000000..6fbf017 --- /dev/null +++ b/src/hooks/trace-window.test.ts @@ -0,0 +1,138 @@ +import { act, renderHook, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as db from '@/lib/trace-db' +import { DEFAULT_SETTINGS, type Settings } from '@/lib/settings' +import { useConsoleUnderTest } from '@/test/console-driver' +import { GO_CONNECT, FIXTURE_NEWEST_EPOCH, traceFrame } from '@/test/stream-fixtures' +import { FakeEventSource, installFakeEventSource, replay } from '@/test/sse-stream' + +/** + * The trace list against the store underneath it. + * + * The list used to only grow: every batch merged in and nothing came out except a + * clear. That cost a sort over the whole session per batch, and — the reason this + * file exists — eviction removed rows from IndexedDB without removing them from the + * list. Measured on the old code with a limit of 8: after ten arrivals the header + * read "15 shown · 8 retained", and 7 of the rows on screen existed nowhere else. + * + * Limits are tiny here for the reason `retention-limit.test.ts` gives: the store is + * limit-agnostic and the dialog's floor is asserted there. + */ + +const withRetention = (retention: number, over: Partial = {}): Settings => ({ + ...DEFAULT_SETTINGS, + retention, + ...over, +}) + +async function settle(): Promise { + await act(async () => { + for (let i = 0; i < 5; i++) await new Promise((r) => setTimeout(r, 0)) + }) +} + +/** Traces in the platform's connect dump — the store's starting count. */ +const ON_CONNECT = 5 + +/** Connect and deliver the fixture dump, all of it persisted. */ +async function connected(settings: Settings) { + const hook = renderHook(() => useConsoleUnderTest(settings)) + await act(async () => { + const es = FakeEventSource.current() + es.emit('open') + replay(es, GO_CONNECT) + }) + await waitFor(async () => expect(await db.countTraces()).toBe(ON_CONNECT)) + return hook +} + +/** Deliver one trace newer than everything so far. */ +async function arrive(n: number): Promise { + await act(async () => { + FakeEventSource.current().emit('trace', traceFrame(9000 + n, FIXTURE_NEWEST_EPOCH + n * 1_000)) + }) + await settle() +} + +let restoreEventSource: () => void + +beforeEach(async () => { + restoreEventSource = installFakeEventSource() + await db.clearTraces() +}) + +afterEach(() => { + restoreEventSource() +}) + +const stored = async () => new Set((await db.getTracePage(1_000)).map((r) => r._key)) + +describe('the list after eviction', () => { + it('never shows a row storage no longer holds', async () => { + const { result } = await connected(withRetention(8)) + for (let n = 1; n <= 10; n++) { + await arrive(n) + await waitFor(async () => expect(result.current.store.total).toBe(await db.countTraces())) + const { rows, total } = result.current.store + expect(rows.length, `after arrival ${n}: ${rows.length} shown · ${total} retained`).toBeLessThanOrEqual(total) + } + const keys = await stored() + const ghosts = result.current.store.rows.filter((r) => !keys.has(r._key)) + expect(ghosts.map((r) => r._key)).toEqual([]) + }) + + it('drops evicted rows from a search result too', async () => { + // A filter is answered from storage, so its result is a list of stored rows — + // and the same eviction takes some of them away. + const { result } = await connected(withRetention(8)) + for (let n = 1; n <= 3; n++) await arrive(n) + act(() => result.current.store.setFilter({ chip: 'all', query: 'hello' })) + await waitFor(() => expect(result.current.store.searching).toBe(false)) + const before = result.current.store.rows.length + + for (let n = 4; n <= 8; n++) await arrive(n) + + const keys = await stored() + expect(result.current.store.rows.filter((r) => !keys.has(r._key)).map((r) => r._key)).toEqual([]) + expect(result.current.store.rows.length).toBeGreaterThan(0) + expect(before).toBeGreaterThan(0) + }) +}) + +describe('the list under steady traffic', () => { + it('holds one page, pushing the oldest out of the bottom', async () => { + const { result } = await connected(withRetention(1_000, { pageSize: 4 })) + for (let n = 1; n <= 6; n++) await arrive(n) + + const { rows } = result.current.store + expect(rows).toHaveLength(4) + // The newest four, in order — the window slides, it does not freeze. + expect(rows.map((r) => r.epochMillis)).toEqual([6, 5, 4, 3].map((n) => FIXTURE_NEWEST_EPOCH + n * 1_000)) + // What left the list is still in storage, and the list says there is more. + expect(await db.countTraces()).toBe(ON_CONNECT + 6) + expect(result.current.store.hasMore).toBe(true) + }) + + it('grows only by what the user loads', async () => { + const { result } = await connected(withRetention(1_000, { pageSize: 4 })) + for (let n = 1; n <= 6; n++) await arrive(n) + + act(() => result.current.store.loadOlder()) + await waitFor(() => expect(result.current.store.rows).toHaveLength(8)) + // The rows the window had pushed out come back from storage, oldest last. + expect(result.current.store.rows[4].epochMillis).toBe(FIXTURE_NEWEST_EPOCH + 2_000) + + // And the deeper window holds through the next arrival instead of snapping back. + await arrive(7) + expect(result.current.store.rows).toHaveLength(8) + expect(result.current.store.rows[0].epochMillis).toBe(FIXTURE_NEWEST_EPOCH + 7_000) + }) + + it('says there is nothing more once storage is exhausted', async () => { + const { result } = await connected(withRetention(1_000, { pageSize: 4 })) + await arrive(1) // 6 stored, 4 shown + act(() => result.current.store.loadOlder()) + await waitFor(() => expect(result.current.store.rows).toHaveLength(6)) + await waitFor(() => expect(result.current.store.hasMore).toBe(false)) + }) +}) diff --git a/src/hooks/use-trace-store.ts b/src/hooks/use-trace-store.ts index 1dca4d2..8627979 100644 --- a/src/hooks/use-trace-store.ts +++ b/src/hooks/use-trace-store.ts @@ -4,9 +4,12 @@ import { countTraces, findTraces, getTracePage, + laterMark, pruneOldest, + survives, traceKey, upsertTraces, + type EvictionMark, type StoredTrace, } from '@/lib/trace-db' import { evictionCount, type Settings } from '@/lib/settings' @@ -33,6 +36,20 @@ function merge(a: StoredTrace[], b: StoredTrace[]): StoredTrace[] { return [...byKey.values()].sort((x, y) => y.epochMillis - x.epochMillis) } +/** + * The rows the list holds: the newest `keep`, minus anything storage evicted. + * + * Both cuts exist because the list used to only grow. Every batch merged in and + * nothing came out except a clear, so a long session sorted hundreds of thousands + * of rows per batch — and eviction removed rows from IndexedDB without removing them + * here, so the header could say "15 shown · 8 retained" over rows that no longer + * existed anywhere but the screen. + */ +function windowed(rows: StoredTrace[], keep: number, evicted: EvictionMark | null): StoredTrace[] { + const kept = evicted === null ? rows : rows.filter((r) => survives(r, evicted)) + return kept.length > keep ? kept.slice(0, keep) : kept +} + /** * Whether the browser's storage is doing what the screen implies it is doing. * @@ -83,6 +100,16 @@ export function useTraceStore(liveTraces: RequestTrace[], persist: boolean, sett settingsRef.current = settings }, [settings]) const [rows, setRows] = useState([]) + // How many rows the list holds: one page, plus a page for every "load older" the + // user asked for. New traffic pushes the oldest out of the bottom rather than + // growing the list; they are still in storage, and loading older brings them back. + // Mirrored in a ref for the async batch path, which must not be re-created by it. + const [keep, setKeepState] = useState(settings.pageSize) + const keepRef = useRef(keep) + const setKeep = useCallback((next: number) => { + keepRef.current = next + setKeepState(next) + }, []) const [total, setTotal] = useState(0) const [hasMore, setHasMore] = useState(false) const [loadingOlder, setLoadingOlder] = useState(false) @@ -175,7 +202,8 @@ export function useTraceStore(liveTraces: RequestTrace[], persist: boolean, sett const page = settingsRef.current.pageSize const first = await getTracePage(page) if (cancelled) return - setRows((prev) => merge(prev, first)) + setKeep(page) + setRows((prev) => windowed(merge(prev, first), page, null)) setHasMore(first.length === page) setTotal(await countTraces()) okStorage('read') @@ -189,7 +217,7 @@ export function useTraceStore(liveTraces: RequestTrace[], persist: boolean, sett return () => { cancelled = true } - }, [persist, failStorage, okStorage]) + }, [persist, failStorage, okStorage, setKeep]) // Live mode: persist + merge each incoming batch. Persistence is best-effort; // the incoming batch is always merged into the view even if IDB write fails. @@ -203,6 +231,8 @@ export function useTraceStore(liveTraces: RequestTrace[], persist: boolean, sett const overtaken = () => cancelled || gen !== clearGenRef.current const batch = fresh.map(withKey) void (async () => { + // How far eviction reached during this batch, across the retry too. + let evicted: EvictionMark | null = null /** * Insert, then evict down to the user's limit. * @@ -218,7 +248,8 @@ export function useTraceStore(liveTraces: RequestTrace[], persist: boolean, sett await upsertTraces(fresh) const after = await countTraces() const s = settingsRef.current - await pruneOldest(evictionCount(after, after - before, s)) + const pruned = await pruneOldest(evictionCount(after, after - before, s)) + evicted = laterMark(evicted, pruned.through) } try { try { @@ -237,7 +268,8 @@ export function useTraceStore(liveTraces: RequestTrace[], persist: boolean, sett const s = settingsRef.current const total = await countTraces() const freed = await pruneOldest(Math.max(fresh.length * s.evictionFactor, total - s.retention)) - if (freed === 0) throw first // nothing to give back; report the original failure + evicted = laterMark(evicted, freed.through) + if (freed.deleted === 0) throw first // nothing to give back; report the original failure await store() } if (overtaken()) return @@ -271,7 +303,14 @@ export function useTraceStore(liveTraces: RequestTrace[], persist: boolean, sett /* leave the count where it was; the write failure is the message that matters */ } } - if (!overtaken()) setRows((prev) => merge(prev, batch)) + if (overtaken()) return + const gone = evicted + setRows((prev) => windowed(merge(prev, batch), keepRef.current, gone)) + // A search result is a list of stored rows too, and the same eviction emptied + // some of them out of storage. + if (gone !== null) { + setResult((prev) => (prev ? { ...prev, rows: prev.rows.filter((r) => survives(r, gone)) } : prev)) + } })() return () => { cancelled = true @@ -349,7 +388,9 @@ export function useTraceStore(liveTraces: RequestTrace[], persist: boolean, sett const current = rowsRef.current const oldest = current[current.length - 1] const page = await getTracePage(size, oldest?.epochMillis) - setRows((prev) => merge(prev, page)) + const next = keepRef.current + size + setKeep(next) + setRows((prev) => windowed(merge(prev, page), next, null)) setHasMore(page.length === size) okStorage('read') } catch (e) { @@ -361,7 +402,7 @@ export function useTraceStore(liveTraces: RequestTrace[], persist: boolean, sett setLoadingOlder(false) } })() - }, [failStorage, okStorage]) + }, [failStorage, okStorage, setKeep]) // Wipe the retained history and the display window, and record how far the wipe // reached so a later batch — or a platform replaying its ring buffer on the next @@ -387,11 +428,12 @@ export function useTraceStore(liveTraces: RequestTrace[], persist: boolean, sett return } okStorage('clear') + setKeep(settingsRef.current.pageSize) setRows([]) setTotal(0) setHasMore(false) })() - }, [failStorage, okStorage]) + }, [failStorage, okStorage, setKeep]) // Sample mode has nothing persisted to wipe, so "clear" records which batch // was dismissed. Storing the input alongside the decision keeps the view @@ -424,8 +466,9 @@ export function useTraceStore(liveTraces: RequestTrace[], persist: boolean, sett return { rows: view, total, - // Under a filter, "more" means more matches to find, not more rows to page. - hasMore: filtering ? moreMatches : hasMore, + // Under a filter, "more" means more matches to find, not more rows to page. A full + // window may have pushed rows out of the bottom, which storage still holds. + hasMore: filtering ? moreMatches : hasMore || rows.length >= keep, loadingOlder, loadOlder, clear, diff --git a/src/lib/trace-db.ts b/src/lib/trace-db.ts index 450a1ae..fc0ee0e 100644 --- a/src/lib/trace-db.ts +++ b/src/lib/trace-db.ts @@ -153,6 +153,34 @@ export async function clearTraces(): Promise { await (await db()).clear(STORE) } +/** + * How far an eviction reached: the last row it deleted, in the order it deleted them. + * + * Rows are evicted oldest-first along `byEpoch`, and an index orders equal keys by + * primary key, so everything at or before this position is gone and everything + * after it survived. The trace list uses it to drop exactly what storage dropped — + * an epoch alone could not, since rows sharing the boundary millisecond may have + * gone either way. + */ +export interface EvictionMark { + epochMillis: number + key: string +} + +/** The row survived an eviction that reached `mark`. */ +export function survives(t: StoredTrace, mark: EvictionMark | null): boolean { + if (mark === null) return true + // IndexedDB compares string keys by UTF-16 code unit, which is what `>` does too. + return t.epochMillis > mark.epochMillis || (t.epochMillis === mark.epochMillis && t._key > mark.key) +} + +/** The later of two marks — what two evictions in a row removed between them. */ +export function laterMark(a: EvictionMark | null, b: EvictionMark | null): EvictionMark | null { + if (a === null) return b + if (b === null) return a + return survives({ epochMillis: b.epochMillis, _key: b.key } as StoredTrace, a) ? b : a +} + /** * Drop the oldest rows once the store is over its limit. * @@ -160,23 +188,27 @@ export async function clearTraces(): Promise { * of the batch that arrived and the user's eviction factor (`evictionCount` in * `settings.ts`). Passing it in rather than recomputing here keeps the policy in * one place and lets this stay a mechanism. + * + * Returns how many went and how far the deletion reached (`null` when none did). */ -export async function pruneOldest(count: number): Promise { - if (count <= 0) return 0 +export async function pruneOldest(count: number): Promise<{ deleted: number; through: EvictionMark | null }> { + if (count <= 0) return { deleted: 0, through: null } const database = await db() const tx = database.transaction(STORE, 'readwrite') observeAbort(tx) let cursor = await tx.store.index(BY_EPOCH).openCursor(null, 'next') // oldest first let deleted = 0 let toDelete = count + let through: EvictionMark | null = null while (cursor && toDelete > 0) { + through = { epochMillis: cursor.value.epochMillis, key: cursor.primaryKey } await cursor.delete() deleted += 1 toDelete -= 1 cursor = await cursor.continue() } await tx.done - return deleted + return { deleted, through } } /** Drop oldest rows beyond `max` (default retention cap). */