From ed0df7bec4aa2a06cc355aeb59a6dfb7acf3e960 Mon Sep 17 00:00:00 2001 From: James Foster Date: Mon, 5 Oct 2026 03:07:57 -0700 Subject: [PATCH 1/5] Show statmon files in GemDB Stats, in an editor tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GemDB Stats' Flutter web build runs in a read-only custom editor for *.out and *.out.gz (Open With…, never the default), and GemDB: Open Statistics File… picks a file with VS Code's own dialog. A webview works under code-server too, where the only screen is a browser tab, and needs no database, so it is registered before the platform gate. scripts/bundle-stats.mjs (npm run bundle:stats, STATS_WEB=) makes a web build hostable: a CSP, a on the webview, a nonce on every script, Flutter's loader without its service worker, and history updates kept off the base's origin -- in a real webview that origin differs from the page's, so Flutter's first replaceState threw and the app stopped at a grey screen. It refuses builds it cannot host, and drops what is never loaded (47 MB to 22.7 MB). The page is handed its file as window.gemdbStatsHost; GemDB Stats does not read it yet. Not wired into CI or check-vsix.sh until there is a pinned source for the build. See docs/statistics.md. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 4 + .prettierignore | 1 + CLAUDE.md | 1 + docs/README.md | 1 + docs/statistics.md | 112 ++++++++++++ eslint.config.mjs | 13 +- package.json | 22 +++ scripts/bundle-stats.mjs | 282 +++++++++++++++++++++++++++++ src/__mocks__/vscode.ts | 11 ++ src/__tests__/activation.test.ts | 12 ++ src/__tests__/manifest.test.ts | 22 +++ src/__tests__/statistics.test.ts | 301 +++++++++++++++++++++++++++++++ src/extension.ts | 6 + src/statistics.ts | 162 +++++++++++++++++ 14 files changed, 948 insertions(+), 2 deletions(-) create mode 100644 docs/statistics.md create mode 100644 scripts/bundle-stats.mjs create mode 100644 src/__tests__/statistics.test.ts create mode 100644 src/statistics.ts diff --git a/.gitignore b/.gitignore index 75cbc9a..45dacef 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,10 @@ grail/ # but it is still the mcp_server repository's code, not this one's. mcp/ +# The GemDB Stats payload is a build artifact of scripts/bundle-stats.mjs: a +# Flutter web build from the GemDB_Stats repository, made hostable in a webview. +stats/ + # The extent the integration suite starts from, built by # scripts/build-test-extent.sh. A TEST artifact: no extent is shipped, because # a user's database is theirs and Grail is installed into it. diff --git a/.prettierignore b/.prettierignore index 10d7f24..c57fc33 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,5 +1,6 @@ out grail +stats *.vsix # Vendored from Jasper byte-for-byte so upstream fixes are a plain copy. diff --git a/CLAUDE.md b/CLAUDE.md index b37e25a..04b2440 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,6 +19,7 @@ npm run test:integration # a real database in a temp root path; seconds npm run bundle # esbuild -> out/extension.js npm run bundle:grail # assemble the Grail payload (needs a C toolchain) npm run bundle:mcp # assemble the MCP server payload (needs nothing) +npm run bundle:stats # make a GemDB Stats web build hostable (STATS_WEB=) npm run test:extent # build the extent the integration suite starts from npm run package # .vsix npm run hooks:uninstall # remove the local git hooks npm install added diff --git a/docs/README.md b/docs/README.md index a956cf0..aa5caad 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,5 +10,6 @@ changing it. | [`automation-boundary.md`](automation-boundary.md) | where each automated step sits on the "inert and reversible vs persistent or global" line, and what was measured about asking for shared memory | adding or changing anything GemDB does without asking | | [`grail.md`](grail.md) | how the Grail payload is built and staged, the Python↔GCI bridge (exit codes, `input()`, `print()`, encoding), and the gotchas found building on it | touching the bridge or building an application on Grail | | [`mcp-server.md`](mcp-server.md) | the bundled MCP server: payload, session cost and leak, read-only mode, stop order, client registration | changing how the MCP server is built, started, stopped or exposed | +| [`statistics.md`](statistics.md) | GemDB Stats in a webview: why a webview, the `gemdbStatsHost` contract with GemDB Stats, what `bundle-stats.mjs` does to a Flutter web build, and what is still unmeasured | changing how statmon files are opened or how the GemDB Stats build is bundled | | [`reaching-windows.md`](reaching-windows.md) | the options for a Windows story (WSL, remote server, Docker) and what still needs a Windows machine | starting any Windows work | | [`demo/`](demo/) | one self-contained directory per demo, with measured findings | building something a demo already explored, or adding a demo | diff --git a/docs/statistics.md b/docs/statistics.md new file mode 100644 index 0000000..4e0c6dc --- /dev/null +++ b/docs/statistics.md @@ -0,0 +1,112 @@ +# GemDB Stats + +GemDB Code shows GemTalk's statmon viewer, +[GemDB Stats](https://github.com/GemTalk/GemDB_Stats), in an editor tab. A +Flutter web build of it ships in the `.vsix` as `stats/`. `src/statistics.ts` +hosts it, and `scripts/bundle-stats.mjs` prepares it. + +## Why a webview + +The editor is not always on the machine that holds the statmon files. Under +code-server the only screen is a browser tab, so a desktop app cannot appear +there. A webview is an iframe in that tab whose files the remote machine +serves, so one build works in desktop VS Code and in hosted VS Code. It needs +no port, no server process and no login of its own. + +Viewing statistics needs no database. The editor is registered before the +platform gate, so it works on machines that cannot host a database. + +## How a file reaches the page + +- **Custom editor** `gemdb.statistics`, read-only, for `*.out` and `*.out.gz`. + Its priority is `option`, so it appears under **Open With…** and never opens + a `.out` file on a click. `.out` is too common an extension to take over. +- **GemDB: Open Statistics File…** picks a file and opens it in that editor. +- **The page's resource roots** are the build and the folder holding the file. + The page fetches the file from the webview's resource origin, the same way + it fetches its own code. Its CSP allows nothing else. +- **`retainContextWhenHidden`** keeps a hidden tab alive. Bringing a discarded + one back means starting Flutter and parsing the file again. + +## The contract with GemDB Stats + +Before any of the page's own scripts run, the host sets: + +```js +window.gemdbStatsHost = { file: { url: '', name: 'statmon76637.out' } }; +``` + +The app reads it at startup and fetches `file.url`. The response is the +file's bytes exactly as they are on disk, so a `.out.gz` arrives +gzip-compressed. Without `gemdbStatsHost`, for example on GitHub Pages, the app +shows its own file picker. + +The build must be: + +- dart2js with the CanvasKit renderer, which is `flutter build web` without + `--wasm`; +- built with `--no-web-resources-cdn`, because the CSP refuses `gstatic.com`; +- an `index.html` with one `` and one `flutter_bootstrap.js` + script tag. + +`bundle-stats.mjs` refuses any build that is not all of these. + +## What `bundle-stats.mjs` changes + +`bundle-stats.mjs` turns `index.html` into `host.html`, a template: + +- It adds a CSP, first in ``. +- It points `` at the webview. +- It puts a nonce on every script. +- It replaces `flutter_bootstrap.js` with a direct call to Flutter's loader. + The stock bootstrap registers a service worker, which a webview cannot host. +- It keeps Flutter's `history.pushState` and `replaceState` calls on the + page's own URL. Flutter resolves the URL against ``, which in a + webview is another origin, so the browser refuses the call and Flutter stops + before drawing. A webview has no address bar, so dropping the URL costs + nothing. + +The page keeps everything else, including its loading screen, which belongs to +GemDB Stats. `fillHostPage` in `statistics.ts` fills the four `{{GEMDB_*}}` +placeholders for each panel. + +These changes happen at bundle time, not when the panel opens, so a build +GemDB Code cannot host fails in front of whoever is packaging it. + +`bundle-stats.mjs` also leaves out files that are never loaded: + +- the Wasm renderers; +- debug symbols; +- the service worker; +- the PWA manifest and icons. + +That cuts the 47 MB build to a 22.7 MB payload. + +## Measured + +On 2026-10-04, in headless Chrome 154: the generated page, served the way a +webview serves it, with the full CSP. Flutter drew its first frame, there were +no CSP violations, no service worker was registered, and a fetch of a statmon +file from the page succeeded. + +That first test served the page and the build from one origin, which a +webview does not. In VS Code on 2026-10-05 the app showed its loading screen, +then grey. Chrome reproduced it with the page and the build on different +origins: a SecurityError from Flutter's first `replaceState`. With the history +calls kept on the page's own URL, Chrome draws the app in that same setup. + +## Open + +- **Not yet seen in VS Code or code-server.** The Chrome measurement covers the + CSP, `` and the loader. A real webview adds its own resource + origin and service worker. +- **Large files through VS Code's resource loader.** A 216 MB `.out.gz` takes + 23 s to parse in Chrome when GemDB Stats loads it from its own picker. + Whether the webview resource loader streams a file that size has not been + measured. +- **Where CI gets the build.** `bundle:stats` takes a local build + (`STATS_WEB`). There is no pinned source yet, so `check-vsix.sh` does not + require `stats/` and CI does not run `bundle:stats`. +- **Loading the file from `gemdbStatsHost`** is GemDB Stats' half of the + contract, and is not built yet. Until it is, the panel shows the app with its + own picker. diff --git a/eslint.config.mjs b/eslint.config.mjs index 3521349..e00f4a0 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -10,10 +10,19 @@ export default tseslint.config( // it. The rest are build artifacts that ESLint would otherwise walk on // its own, since (unlike Prettier 3) it does not read .gitignore: out/** // is this project's own bundle, grail/** and mcp/** are third-party - // payloads staged by bundle-grail.sh and bundle-mcp.sh, dist/** is + // payloads staged by bundle-grail.sh and bundle-mcp.sh, stats/** is the + // GemDB Stats web build staged by bundle-stats.mjs, dist/** is // packages fetched from CI by fetch-vsix.sh, and .test-extent/** is the // database extent build-test-extent.sh builds for the integration suite. - ignores: ['out/**', 'grail/**', 'mcp/**', 'dist/**', '.test-extent/**', 'src/gci/**'], + ignores: [ + 'out/**', + 'grail/**', + 'mcp/**', + 'stats/**', + 'dist/**', + '.test-extent/**', + 'src/gci/**', + ], }, js.configs.recommended, ...tseslint.configs.recommended, diff --git a/package.json b/package.json index 0598865..aa2c953 100644 --- a/package.json +++ b/package.json @@ -202,6 +202,21 @@ "when": "gemdb.running && gemdb.hasSavedStacks" } ], + "customEditors": [ + { + "viewType": "gemdb.statistics", + "displayName": "GemDB Stats", + "selector": [ + { + "filenamePattern": "*.out" + }, + { + "filenamePattern": "*.out.gz" + } + ], + "priority": "option" + } + ], "commands": [ { "command": "gemdb.install", @@ -298,6 +313,12 @@ "category": "GemDB", "icon": "$(output)" }, + { + "command": "gemdb.openStatistics", + "title": "Open Statistics File…", + "category": "GemDB", + "icon": "$(graph-line)" + }, { "command": "gemdb.uninstall", "title": "Uninstall GemDB", @@ -679,6 +700,7 @@ "test:integration": "vitest run --config vitest.integration.config.mts", "bundle:grail": "./scripts/bundle-grail.sh", "bundle:mcp": "./scripts/bundle-mcp.sh", + "bundle:stats": "node scripts/bundle-stats.mjs", "test:extent": "./scripts/build-test-extent.sh", "//hooks": "Local git hooks (lefthook.yml): eslint + prettier on staged files, and a non-blocking package-lock.json warning after checkout/merge/rebase. lefthook's postinstall installs them on `npm install`, so use hooks:uninstall to opt out. CI's `checks` job is the gate regardless.", "hooks:install": "lefthook install", diff --git a/scripts/bundle-stats.mjs b/scripts/bundle-stats.mjs new file mode 100644 index 0000000..8aa9653 --- /dev/null +++ b/scripts/bundle-stats.mjs @@ -0,0 +1,282 @@ +#!/usr/bin/env node +// +// Assemble the GemDB Stats payload that ships inside the .vsix. +// +// GemDB Stats is GemTalk's statmon viewer, a Flutter app with a web build. +// GemDB Code shows that build in a webview (src/statistics.ts), so a statmon +// file can be charted wherever the editor is -- including code-server, where +// the only screen is a browser tab and a desktop app cannot appear. +// +// Like Grail and the MCP server, the payload is a build artifact of another +// repository. Unlike them it is not built here: compiling it needs the Flutter +// SDK, so this script takes a finished `flutter build web` output and turns it +// into something a webview can host. +// +// Usage: +// STATS_WEB=/path/to/GemDB_Stats/app/build/web npm run bundle:stats +// +// Environment: +// STATS_WEB the `flutter build web` output to bundle (required) +// STATS_OUT where to assemble the payload (default: stats/, which is what +// the .vsix ships; the unit tests point it elsewhere) +// +// What it does to the build, and why each step happens HERE rather than when +// the panel opens: everything below can fail, and a failure here is in front +// of whoever is packaging, while the same failure at run time is a blank panel +// on a user's screen. +// +// - The page. A webview needs a Content-Security-Policy, a that +// points at the webview's own resource URL, and a nonce on every script. +// That base is on another origin from the page, so Flutter's history updates +// have to be kept off it (HISTORY_SHIM). And +// Flutter's stock bootstrap registers a service worker, which a webview +// cannot host, so the bootstrap is replaced by a direct call to Flutter's +// loader. The build's own index.html is kept for everything else -- its +// loading screen is GemDB Stats' to design -- and becomes `host.html`, a +// template whose four `{{GEMDB_*}}` placeholders statistics.ts fills. +// - The files. Only the dart2js build with the CanvasKit renderer is kept, +// which is what `flutter build web` without `--wasm` produces. The Wasm +// renderers, debug symbols, the service worker and the PWA manifest go: +// more than half of the build's 47 MB, none of it ever loaded here. + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +/** What the hosted page loads. A build missing any of these is not one this can host. */ +const REQUIRED = [ + 'index.html', + 'flutter.js', + 'flutter_bootstrap.js', + 'main.dart.js', + 'version.json', + 'assets/AssetManifest.bin.json', + 'assets/FontManifest.json', + // Flutter's loader picks the Chromium variant where the browser supports it + // (VS Code's own window, Chrome, Edge) and the full one elsewhere (Firefox + // or Safari pointed at code-server), so both stay. + 'canvaskit/canvaskit.js', + 'canvaskit/canvaskit.wasm', + 'canvaskit/chromium/canvaskit.js', + 'canvaskit/chromium/canvaskit.wasm', +]; + +/** + * Paths, relative to the build, that never ship. Everything else does: GemDB + * Stats decides what its build contains, and a list of what to KEEP would + * silently drop the next asset it adds. + */ +function isLeftOut(relative) { + const name = path.posix.basename(relative); + return ( + name.endsWith('.symbols') || + // The Wasm renderers. Only a `--wasm` build loads them. + /^canvaskit\/(skwasm|wimp)/.test(relative) || + // Loaded only when the build config sets `preferWebParagraph`, which + // buildConfigFrom refuses. + relative.startsWith('canvaskit/webparagraph/') || + // Replaced: host.html takes index.html's place, and calls the loader + // itself instead of running the bootstrap. + relative === 'index.html' || + relative === 'flutter_bootstrap.js' || + relative === 'flutter_service_worker.js' || + // A PWA's install metadata. A webview cannot install anything. + relative === 'manifest.json' || + relative.startsWith('icons/') || + relative === '.last_build_id' + ); +} + +/** + * The `_flutter.buildConfig` that `flutter build web` writes into + * flutter_bootstrap.js, reduced to the one build this payload keeps. + */ +function buildConfigFrom(bootstrap) { + const match = /^_flutter\.buildConfig = (\{.*\});$/m.exec(bootstrap); + if (!match) { + throw new Error('flutter_bootstrap.js has no `_flutter.buildConfig = {...};` line.'); + } + const config = JSON.parse(match[1]); + // Without this the loader fetches CanvasKit from www.gstatic.com, which the + // page's CSP refuses -- and which a server with no internet access could not + // reach anyway. + if (config.useLocalCanvasKit !== true) { + throw new Error( + 'The build loads CanvasKit from a CDN. Build it with `--no-web-resources-cdn`.', + ); + } + if (config.canvasKitVariant !== undefined || config.preferWebParagraph !== undefined) { + throw new Error( + 'The build chooses its own CanvasKit variant, which may need files this payload leaves out.', + ); + } + const builds = (config.builds ?? []).filter( + (build) => build.compileTarget === 'dart2js' && build.renderer === 'canvaskit', + ); + if (builds.length !== 1 || builds[0].mainJsPath !== 'main.dart.js') { + throw new Error('The build has no dart2js + CanvasKit entry point at main.dart.js.'); + } + return { ...config, builds }; +} + +/** The `\n' + + ' ' + ); +} + +/** + * The policy the page runs under. `{{GEMDB_CSP_SOURCE}}` becomes the + * webview's own resource origin, which serves both the build and the statmon + * file, so nothing else is reachable. + */ +const CSP = [ + "default-src 'none'", + // CanvasKit is WebAssembly, which a CSP blocks unless this allows it. + "script-src 'nonce-{{GEMDB_NONCE}}' {{GEMDB_CSP_SOURCE}} 'wasm-unsafe-eval'", + // Flutter writes its own + + +
Loading…
+ + + + +`; + +const BUILD_CONFIG = { + engineRevision: 'af7e796e', + builds: [{ compileTarget: 'dart2js', renderer: 'canvaskit', mainJsPath: 'main.dart.js' }, {}], + useLocalCanvasKit: true, +}; + +function bootstrap(config: unknown): string { + return [ + '/* flutter.js */', + '', + `_flutter.buildConfig = ${JSON.stringify(config)};`, + '', + '_flutter.loader.load({', + ' serviceWorkerSettings: { serviceWorkerVersion: "1105577256" }', + '});', + ].join('\n'); +} + +let dir: string; +let web: string; +let out: string; + +function write(relative: string, content = 'x'): void { + const file = path.join(web, relative); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, content); +} + +function bundle(): { status: number | null; stderr: string } { + const result = spawnSync(process.execPath, [SCRIPT], { + env: { ...process.env, STATS_WEB: web, STATS_OUT: out }, + encoding: 'utf8', + }); + return { status: result.status, stderr: result.stderr }; +} + +const shipped = (relative: string) => fs.existsSync(path.join(out, relative)); +const hostHtml = () => fs.readFileSync(path.join(out, 'host.html'), 'utf8'); + +beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gemdb-stats-')); + web = path.join(dir, 'web'); + out = path.join(dir, 'out'); + write('index.html', INDEX_HTML); + write('flutter_bootstrap.js', bootstrap(BUILD_CONFIG)); + write('version.json', JSON.stringify({ app_name: 'vsd', version: '0.1.0' })); + for (const file of [ + 'flutter.js', + 'main.dart.js', + 'favicon.png', + 'logo.svg', + 'manifest.json', + 'flutter_service_worker.js', + '.last_build_id', + 'icons/Icon-192.png', + 'assets/AssetManifest.bin.json', + 'assets/FontManifest.json', + 'assets/fonts/MaterialIcons-Regular.otf', + 'canvaskit/canvaskit.js', + 'canvaskit/canvaskit.js.symbols', + 'canvaskit/canvaskit.wasm', + 'canvaskit/chromium/canvaskit.js', + 'canvaskit/chromium/canvaskit.wasm', + 'canvaskit/skwasm.wasm', + 'canvaskit/wimp.wasm', + 'canvaskit/webparagraph/canvaskit.wasm', + ]) { + write(file); + } +}); + +afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe('bundle-stats.mjs', () => { + it('ships what the dart2js build loads, and none of what it never does', () => { + expect(bundle().status).toBe(0); + + for (const file of [ + 'host.html', + 'flutter.js', + 'main.dart.js', + 'logo.svg', + 'assets/fonts/MaterialIcons-Regular.otf', + 'canvaskit/canvaskit.wasm', + 'canvaskit/chromium/canvaskit.wasm', + ]) { + expect(shipped(file), file).toBe(true); + } + for (const file of [ + 'index.html', + 'flutter_bootstrap.js', + 'flutter_service_worker.js', + 'manifest.json', + 'icons/Icon-192.png', + '.last_build_id', + 'canvaskit/canvaskit.js.symbols', + 'canvaskit/skwasm.wasm', + 'canvaskit/wimp.wasm', + 'canvaskit/webparagraph/canvaskit.wasm', + ]) { + expect(shipped(file), file).toBe(false); + } + }); + + it('puts the policy first in , before any script', () => { + bundle(); + const html = hostHtml(); + + const head = html.indexOf(''); + expect(html.indexOf('Content-Security-Policy')).toBeGreaterThan(head); + expect(html.indexOf('Content-Security-Policy')).toBeLessThan(html.indexOf(' at the webview, and drops the manifest the policy would refuse', () => { + bundle(); + const html = hostHtml(); + + expect(html).toContain(''); + expect(html).not.toContain(''); + expect(html).not.toContain('manifest.json'); + }); + + it('starts Flutter without the bootstrap, so no service worker is registered', () => { + bundle(); + const html = hostHtml(); + + expect(html).not.toContain('flutter_bootstrap.js'); + expect(html).not.toContain('serviceWorker'); + expect(html).toContain(''); + expect(html).toContain("_flutter.loader.load({ nonce: '{{GEMDB_NONCE}}' });"); + }); + + it('keeps only the dart2js build in the loader’s config', () => { + bundle(); + const config = /_flutter\.buildConfig = (\{.*\});/.exec(hostHtml())?.[1]; + + expect(JSON.parse(config ?? 'null')).toEqual({ + ...BUILD_CONFIG, + builds: [BUILD_CONFIG.builds[0]], + }); + }); + + it('puts the nonce on every script, the page’s own included', () => { + bundle(); + const scripts = hostHtml().match(/]*>/g) ?? []; + + // The host's settings, the loading screen's, and the loader's two. + expect(scripts).toHaveLength(4); + expect(scripts.every((tag) => tag.includes('nonce="{{GEMDB_NONCE}}"'))).toBe(true); + }); + + it('keeps Flutter’s history updates on the page’s own origin', () => { + // The base is the webview's resource origin, so a URL Flutter resolves + // against it is cross-origin, and the browser refuses it -- which stops + // the app before its first frame. + bundle(); + const html = hostHtml(); + const shim = html.indexOf("for (const name of ['pushState', 'replaceState'])"); + + expect(shim).toBeGreaterThan(-1); + expect(shim).toBeLessThan(html.indexOf('src="flutter.js"')); + }); + + it('refuses a build that loads CanvasKit from a CDN', () => { + write('flutter_bootstrap.js', bootstrap({ ...BUILD_CONFIG, useLocalCanvasKit: false })); + + const result = bundle(); + + expect(result.status).toBe(1); + expect(result.stderr).toContain('--no-web-resources-cdn'); + expect(shipped('host.html')).toBe(false); + }); + + it('refuses a Wasm-only build, whose renderer it leaves out', () => { + write( + 'flutter_bootstrap.js', + bootstrap({ ...BUILD_CONFIG, builds: [{ compileTarget: 'dart2wasm', renderer: 'skwasm' }] }), + ); + + expect(bundle().status).toBe(1); + }); + + it('names what a build is missing', () => { + fs.rmSync(path.join(web, 'canvaskit/chromium/canvaskit.wasm')); + + const result = bundle(); + + expect(result.status).toBe(1); + expect(result.stderr).toContain('canvaskit/chromium/canvaskit.wasm'); + }); + + it('refuses a page it cannot find the bootstrap in', () => { + write('index.html', INDEX_HTML.replace(/', { + ...values, + host: { file: { url: 'U', name: '' } }, + }); + + expect(page.match(/<\/script>/g)).toHaveLength(1); + expect(page).toContain('\\u003c/script>'); + }); + + it('does not fill a placeholder that arrives inside a value', () => { + const page = fillHostPage('{{GEMDB_HOST}}', { + ...values, + host: { file: { url: 'U', name: '{{GEMDB_NONCE}}' } }, + }); + + expect(page).toContain('{{GEMDB_NONCE}}'); + }); +}); + +describe('unavailablePage', () => { + it('shows its message as text', () => { + expect(unavailablePage('N', 'Run `a `')).toContain('Run `a <b>`'); + }); +}); diff --git a/src/extension.ts b/src/extension.ts index 301f5c4..b9f079b 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -53,6 +53,7 @@ import { debugFile } from './debugFile'; import { openRepl, runFile } from './repl'; import { closeSessionFor, logoutAll, setInputHandler } from './session'; import { GemDbStatusBar } from './statusBar'; +import { registerStatistics } from './statistics'; import { StatusViewProvider } from './statusView'; import { SETUP_OUTCOME, @@ -124,6 +125,11 @@ export function activate(context: vscode.ExtensionContext): void { registerMcpProvider(extensionPath, () => ensureMcpRunning(extensionPath)), ); + // GemDB Stats charts a statmon file in an editor tab. Before the platform + // gate: it reads a file and needs no database, so it works on a machine + // that cannot host one. + context.subscriptions.push(registerStatistics(context.extensionUri)); + // A closed notebook gives its session back. Sessions are scarce — the // database allows ten at once, and its own gems spend some of that — so a // window that opened and closed notebooks all morning must not still be diff --git a/src/statistics.ts b/src/statistics.ts new file mode 100644 index 0000000..3eb026b --- /dev/null +++ b/src/statistics.ts @@ -0,0 +1,162 @@ +import * as crypto from 'crypto'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as vscode from 'vscode'; +import { errorMessage, log } from './log'; + +/** + * GemDB Stats in an editor tab: a statmon file charted by GemTalk's statmon + * viewer, whose Flutter web build ships in the .vsix as `stats/`. + * + * It is a webview rather than the desktop app because the editor is not always + * on the machine with the files. Under code-server the only screen is a + * browser tab, and a webview is an iframe in that tab whose files are served + * by the remote machine — so the same panel works wherever the editor does, + * with no port, no server process and no login of its own. The page fetches + * the statmon file from the webview's resource origin, the same way it + * fetches its own code. + * + * Viewing statistics needs no database, so none of this waits for one, and it + * works on platforms GemDB cannot host a database on. + * + * `scripts/bundle-stats.mjs` turns GemDB Stats' index.html into `host.html`, + * a template carrying the page's CSP and a loader that registers no service + * worker. What is left for here is what differs per panel: the four + * `{{GEMDB_*}}` placeholders. + */ + +/** The custom editor's view type, as package.json declares it. */ +export const STATISTICS_VIEW_TYPE = 'gemdb.statistics'; + +/** + * What the page is told about its host, as `window.gemdbStatsHost`. This is + * the contract with GemDB Stats: the app reads it at startup and fetches + * `file.url`, which serves the file's bytes exactly as they are on disk — + * gzip-compressed for a `.out.gz`. + */ +export interface StatsHost { + file: { url: string; name: string }; +} + +export interface HostPageValues { + /** The build's own resource URL, ending in `/`; relative URLs resolve against it. */ + base: string; + /** The webview's resource origin, which serves the build and the file. */ + cspSource: string; + /** Fresh for every page, so nothing injected into the page can guess it. */ + nonce: string; + host: StatsHost; +} + +/** `host.html` with this panel's values in place of its placeholders. */ +export function fillHostPage(template: string, values: HostPageValues): string { + // `<` is escaped so a file named `` cannot end the script it sits in. + const host = JSON.stringify(values.host).replace(/ = { + '{{GEMDB_BASE}}': values.base, + '{{GEMDB_CSP_SOURCE}}': values.cspSource, + '{{GEMDB_NONCE}}': values.nonce, + '{{GEMDB_HOST}}': host, + }; + // One pass, so a value that happens to contain a placeholder is not filled + // in turn. + return template.replace( + /\{\{GEMDB_[A-Z_]+\}\}/g, + (placeholder) => fills[placeholder] ?? placeholder, + ); +} + +/** A page that says why there is no chart, in the editor's own colours. */ +export function unavailablePage(nonce: string, message: string): string { + const text = message.replace(/[&<>"]/g, (c) => `&#${c.charCodeAt(0)};`); + return ` + + + + + + +

${text}

+`; +} + +class StatisticsEditorProvider implements vscode.CustomReadonlyEditorProvider { + constructor(private readonly extensionUri: vscode.Uri) {} + + openCustomDocument(uri: vscode.Uri): vscode.CustomDocument { + return { uri, dispose: () => {} }; + } + + resolveCustomEditor(document: vscode.CustomDocument, panel: vscode.WebviewPanel): void { + const webview = panel.webview; + const build = vscode.Uri.joinPath(this.extensionUri, 'stats'); + webview.options = { + enableScripts: true, + // The build, and the folder holding the file. A root is a folder, so the + // page could fetch the file's neighbours too — read-only, and only what + // the user's own editor could already open. + localResourceRoots: [build, vscode.Uri.joinPath(document.uri, '..')], + }; + const nonce = crypto.randomBytes(16).toString('base64'); + let template: string; + try { + template = fs.readFileSync(path.join(build.fsPath, 'host.html'), 'utf8'); + } catch (error) { + log(`GemDB Stats is not in this build: ${errorMessage(error)}`); + webview.html = unavailablePage( + nonce, + 'This build of GemDB Code does not include GemDB Stats. ' + + 'A development build gets it from `npm run bundle:stats`.', + ); + return; + } + webview.html = fillHostPage(template, { + base: `${webview.asWebviewUri(build).toString()}/`, + cspSource: webview.cspSource, + nonce, + host: { + file: { + url: webview.asWebviewUri(document.uri).toString(), + name: path.posix.basename(document.uri.path), + }, + }, + }); + } +} + +/** Open Statistics File…: the file the command was given, or one the user picks. */ +async function openStatistics(uri?: vscode.Uri): Promise { + const target = + uri ?? + ( + await vscode.window.showOpenDialog({ + title: 'Open Statistics File', + openLabel: 'Open', + canSelectMany: false, + filters: { 'statmon files': ['out', 'gz'], 'All files': ['*'] }, + }) + )?.[0]; + if (!target) return; + await vscode.commands.executeCommand('vscode.openWith', target, STATISTICS_VIEW_TYPE); +} + +export function registerStatistics(extensionUri: vscode.Uri): vscode.Disposable { + return vscode.Disposable.from( + vscode.window.registerCustomEditorProvider( + STATISTICS_VIEW_TYPE, + new StatisticsEditorProvider(extensionUri), + { + // A hidden webview is discarded by default, and bringing it back would + // start Flutter and parse the file again — seconds for a small file, + // tens of seconds for a large one. Kept, at the cost of its memory. + webviewOptions: { retainContextWhenHidden: true }, + supportsMultipleEditorsPerDocument: true, + }, + ), + vscode.commands.registerCommand('gemdb.openStatistics', (uri?: vscode.Uri) => + openStatistics(uri), + ), + ); +} From 981c3667fb946e4a0a3c734b5b259518b5ad586a Mon Sep 17 00:00:00 2001 From: James Foster Date: Mon, 5 Oct 2026 21:01:32 -0700 Subject: [PATCH 2/5] Open statmon files in GemDB Stats, using GemDB Stats' message protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GemDB Stats PR #10 defines how a host page chooses the file: the app says `ready`, the host answers `open` with the file's webview URL, and a click on the file name asks for another with `pickFile`. GemDB Code now speaks it, and the window.gemdbStatsHost global is gone. - A WebviewPanel replaces the custom editor, since a later `open` replaces the file in the same tab. GemDB: Open Statistics File… shows VS Code's dialog first; Open in GemDB Stats is on .out / .out.gz in the explorer. `pickFile` is answered with VS Code's dialog, which under code-server browses the server rather than the user's laptop. A file in a new folder widens localResourceRoots, and since `ready` is answered every time, a page the change reloads gets its file again. - The history shim and the replacement loader are gone: PR #10 fixed the cross-origin grey screen in the app and starts Flutter without a service worker. bundle-stats.mjs now keeps GemDB Stats' own bootstrap and refuses one that registers a service worker, a Wasm build, or a CDN CanvasKit. - The web build comes from a GemDB Stats release asset, GemDB-Stats--web.tar.gz, pinned by URL and SHA-256 in vendor-pins.sh, so CI needs no Flutter SDK. bundle:stats downloads and checks it (STATS_WEB still takes a local build), each integration leg runs it, and check-vsix.sh requires the payload. The pins are empty until GemDB Stats publishes a release with the web build, so the integration legs fail at bundle:stats until then. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 7 + CHANGELOG.md | 6 + CLAUDE.md | 2 +- docs/README.md | 2 +- docs/statistics.md | 151 ++++++++---------- package.json | 32 ++-- scripts/bundle-stats.mjs | 263 ++++++++++++++++++------------- scripts/check-vsix.sh | 12 +- src/__mocks__/vscode.ts | 8 - src/__tests__/activation.test.ts | 3 +- src/__tests__/manifest.test.ts | 30 ++-- src/__tests__/statistics.test.ts | 190 +++++++++++++--------- src/statistics.ts | 205 +++++++++++++++--------- vendor-pins.sh | 17 +- 14 files changed, 544 insertions(+), 384 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7397def..4d8cc24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -246,6 +246,13 @@ jobs: MCP_REF: ${{ inputs.mcp-ref }} run: npm run bundle:mcp + # The GemDB Stats web build, downloaded from the GemDB Stats release + # pinned in vendor-pins.sh and refused unless its SHA-256 matches. No + # Flutter SDK here: GemDB Stats builds it and publishes it. Platform- + # neutral like the MCP payload, and fetched per leg for the same reason. + - name: Fetch the GemDB Stats web build + run: npm run bundle:stats + # Not a shipped artifact: GemDB installs Grail into whatever database the # user already has, so nothing here ships an extent. This one exists so # the tests that need Python but are not testing the file-in do not pay diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bb16ff..9d25c38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ this one does not look. ### Added +- **Statmon files open in GemDB Stats, inside the editor.** Right-click a `.out` or `.out.gz` in + the explorer and choose **Open in GemDB Stats**, or run **GemDB: Open Statistics File…**, and the + file opens in a tab of GemTalk's statmon viewer: browse processes, plot statistics, compare them + on one chart. Clicking the file name opens another file in the same tab, through VS Code's own + dialog, so under code-server it browses the server where the files are. Large files load in the + background with a progress bar. It needs no database, so it works before GemDB is set up. - **The database reserves the free license's full 10 GB, with room to collect its garbage.** GemDB Code caps the database at the license's 10 GB (until now it stopped at 8 GB), reserves all of it on disk when the database first starts, and keeps 500 MB of it free, so the database protects diff --git a/CLAUDE.md b/CLAUDE.md index 9bcd73e..2e6fb3a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,7 +19,7 @@ npm run test:integration # a real database in a temp root path; seconds npm run bundle # esbuild -> out/extension.js npm run bundle:grail # assemble the Grail payload (needs a C toolchain) npm run bundle:mcp # assemble the MCP server payload (needs nothing) -npm run bundle:stats # make a GemDB Stats web build hostable (STATS_WEB=) +npm run bundle:stats # fetch the pinned GemDB Stats web build (or STATS_WEB=) npm run test:extent # build the extent the integration suite starts from npm run package # .vsix npm run dev:fresh # this checkout in a throwaway profile + empty root path (also :cached-engine, :keep-root) diff --git a/docs/README.md b/docs/README.md index fc67ad3..4fa52af 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,6 @@ changing it. | [`grail.md`](grail.md) | how the Grail payload is built and staged, the Python↔GCI bridge (exit codes, `input()`, `print()`, encoding), and the gotchas found building on it | touching the bridge or building an application on Grail | | [`mcp-server.md`](mcp-server.md) | the bundled MCP server: payload, session cost and leak, read-only mode, stop order, client registration | changing how the MCP server is built, started, stopped or exposed | | [`repository-space.md`](repository-space.md) | the 10 GB cap, reserved on disk, and the free-space threshold; what the stone does below it (stops reclaiming garbage, refuses and ends `gemdb` sessions); the `gemdb` account and what needs DataCurator; why idle sessions are aborted; what a garbage collection has to wait for | changing the stone's space settings, garbage collection, database accounts, or anything that holds a session open | -| [`statistics.md`](statistics.md) | GemDB Stats in a webview: why a webview, the message protocol with GemDB Stats, what `bundle-stats.mjs` does to a Flutter web build, and what is still unmeasured | changing how statmon files are opened or how the GemDB Stats build is bundled | +| [`statistics.md`](statistics.md) | GemDB Stats in a webview: why a webview, the message protocol with GemDB Stats, where the web build comes from and what `bundle-stats.mjs` checks and changes, and what is still unmeasured | changing how statmon files are opened or how the GemDB Stats build is bundled | | [`reaching-windows.md`](reaching-windows.md) | the options for a Windows story (WSL, remote server, Docker) and what still needs a Windows machine | starting any Windows work | | [`demo/`](demo/) | one self-contained directory per demo, with measured findings | building something a demo already explored, or adding a demo | diff --git a/docs/statistics.md b/docs/statistics.md index 4e0c6dc..7e00b15 100644 --- a/docs/statistics.md +++ b/docs/statistics.md @@ -1,9 +1,10 @@ # GemDB Stats -GemDB Code shows GemTalk's statmon viewer, -[GemDB Stats](https://github.com/GemTalk/GemDB_Stats), in an editor tab. A -Flutter web build of it ships in the `.vsix` as `stats/`. `src/statistics.ts` -hosts it, and `scripts/bundle-stats.mjs` prepares it. +GemDB Code shows [GemDB Stats](https://github.com/GemTalk/GemDB_Stats), +GemTalk's statmon viewer, in an editor tab. A Flutter web build of it ships in +the `.vsix` as `stats/`. `src/statistics.ts` hosts it, and +`scripts/bundle-stats.mjs` prepares it. GemDB Stats' side of the arrangement is +its `docs/embedding.md`. ## Why a webview @@ -13,100 +14,88 @@ there. A webview is an iframe in that tab whose files the remote machine serves, so one build works in desktop VS Code and in hosted VS Code. It needs no port, no server process and no login of its own. -Viewing statistics needs no database. The editor is registered before the -platform gate, so it works on machines that cannot host a database. +Viewing statistics needs no database. The commands are registered before the +platform gate, so they work on machines that cannot host a database. -## How a file reaches the page +## Opening a file -- **Custom editor** `gemdb.statistics`, read-only, for `*.out` and `*.out.gz`. - Its priority is `option`, so it appears under **Open With…** and never opens - a `.out` file on a click. `.out` is too common an extension to take over. -- **GemDB: Open Statistics File…** picks a file and opens it in that editor. -- **The page's resource roots** are the build and the folder holding the file. - The page fetches the file from the webview's resource origin, the same way - it fetches its own code. Its CSP allows nothing else. -- **`retainContextWhenHidden`** keeps a hidden tab alive. Bringing a discarded - one back means starting Flutter and parsing the file again. +- **GemDB: Open Statistics File…** shows VS Code's open dialog, then opens a + tab on the file. +- **Open in GemDB Stats** opens a `.out` or `.out.gz` from the explorer's + right-click menu. +- A file already open in a tab is brought forward, not parsed a second time. +- Each tab is a `WebviewPanel` with `retainContextWhenHidden`. Bringing a + discarded tab back would mean starting Flutter and parsing the file again. -## The contract with GemDB Stats +## The protocol -Before any of the page's own scripts run, the host sets: +Three messages, as GemDB Stats' `docs/embedding.md` specifies: -```js -window.gemdbStatsHost = { file: { url: '', name: 'statmon76637.out' } }; -``` +| Message | Direction | GemDB Code's part | +| --- | --- | --- | +| `{type: 'ready'}` | app → GemDB Code | Answer with `open` for the tab's file, every time: a reloaded page says `ready` again. | +| `{type: 'open', url, name}` | GemDB Code → app | `url` is `asWebviewUri(file)`. A later `open` replaces the file. | +| `{type: 'pickFile'}` | app → GemDB Code | The user clicked the file name. Show VS Code's open dialog, then `open` the file in the same tab. | -The app reads it at startup and fetches `file.url`. The response is the -file's bytes exactly as they are on disk, so a `.out.gz` arrives -gzip-compressed. Without `gemdbStatsHost`, for example on GitHub Pages, the app -shows its own file picker. +`pickFile` uses VS Code's dialog rather than the browser's because under +code-server the browser's dialog browses the user's own machine, not the +server where the statmon files are. -The build must be: +The page can fetch only under the panel's `localResourceRoots`: the build and +the folder of every file opened in that tab. A picked file in a new folder +adds its folder, and changing the roots reloads the page. The reloaded page +says `ready` and gets its file then, so the extension does not depend on +whether the reload happens. -- dart2js with the CanvasKit renderer, which is `flutter build web` without - `--wasm`; -- built with `--no-web-resources-cdn`, because the CSP refuses `gstatic.com`; -- an `index.html` with one `` and one `flutter_bootstrap.js` - script tag. +## The build -`bundle-stats.mjs` refuses any build that is not all of these. +A GemDB Stats release publishes `GemDB-Stats--web.tar.gz` beside its +desktop builds. `vendor-pins.sh` pins its URL and SHA-256, and +`npm run bundle:stats` downloads it, checks it and assembles `stats/`. GemDB +Code's CI does not carry the Flutter SDK, which is why it uses a release asset +rather than building a pinned commit, as Grail and the MCP server do. For a +local build, set `STATS_WEB` to `GemDB_Stats/app/build/web`. -## What `bundle-stats.mjs` changes +`bundle-stats.mjs` refuses a build a webview cannot run: -`bundle-stats.mjs` turns `index.html` into `host.html`, a template: +- one that loads CanvasKit from Google's CDN, which the CSP refuses: build with + `--no-web-resources-cdn`; +- one that offers a Wasm build, whose renderers the payload leaves out: build + without `--wasm`; +- one whose bootstrap registers a service worker, which a webview cannot host. + GemDB Stats' own bootstrap template registers none, from PR #10 on. -- It adds a CSP, first in ``. -- It points `` at the webview. -- It puts a nonce on every script. -- It replaces `flutter_bootstrap.js` with a direct call to Flutter's loader. - The stock bootstrap registers a service worker, which a webview cannot host. -- It keeps Flutter's `history.pushState` and `replaceState` calls on the - page's own URL. Flutter resolves the URL against ``, which in a - webview is another origin, so the browser refuses the call and Flutter stops - before drawing. A webview has no address bar, so dropping the URL costs - nothing. +Its changes to the page are few: a CSP first in ``, `` +pointing at the webview, and a nonce on every script. They are made at bundle +time, into a `host.html` template, so a page GemDB Code cannot host fails in +front of whoever is packaging it. `fillHostPage` in `statistics.ts` fills the +three `{{GEMDB_*}}` placeholders for each panel. -The page keeps everything else, including its loading screen, which belongs to -GemDB Stats. `fillHostPage` in `statistics.ts` fills the four `{{GEMDB_*}}` -placeholders for each panel. - -These changes happen at bundle time, not when the panel opens, so a build -GemDB Code cannot host fails in front of whoever is packaging it. - -`bundle-stats.mjs` also leaves out files that are never loaded: - -- the Wasm renderers; -- debug symbols; -- the service worker; -- the PWA manifest and icons. - -That cuts the 47 MB build to a 22.7 MB payload. +It also leaves out what a webview never loads: the Wasm renderers, debug +symbols, the service worker, and the PWA manifest and icons. That cuts the +47 MB build to about 23 MB. `check-vsix.sh` asserts the payload is present and +those files are not. ## Measured -On 2026-10-04, in headless Chrome 154: the generated page, served the way a -webview serves it, with the full CSP. Flutter drew its first frame, there were -no CSP violations, no service worker was registered, and a fetch of a statmon -file from the page succeeded. - -That first test served the page and the build from one origin, which a -webview does not. In VS Code on 2026-10-05 the app showed its loading screen, -then grey. Chrome reproduced it with the page and the build on different -origins: a SecurityError from Flutter's first `replaceState`. With the history -calls kept on the page's own URL, Chrome draws the app in that same setup. +- **2026-10-04, headless Chrome 154.** The generated page, with the full CSP. + Flutter drew its first frame, there were no CSP violations and no service + worker, and the page could fetch a statmon file. +- **2026-10-05, VS Code.** The app showed its loading screen, then grey. + Chrome reproduced it once the page and the build were on different origins, + as they are in a webview: Flutter's first `history.replaceState` resolved its + URL against `` and threw a SecurityError. The first test used one + origin and could not see it. GemDB Stats fixed it at the source in PR #10: + the app sets no URL strategy, so Flutter never touches the history. ## Open -- **Not yet seen in VS Code or code-server.** The Chrome measurement covers the - CSP, `` and the loader. A real webview adds its own resource - origin and service worker. +- **No GemDB Stats release has the web build yet.** Until one does, the pins in + `vendor-pins.sh` are empty, `bundle:stats` needs `STATS_WEB`, and the CI + integration legs fail at it. +- **Not yet seen in code-server.** - **Large files through VS Code's resource loader.** A 216 MB `.out.gz` takes - 23 s to parse in Chrome when GemDB Stats loads it from its own picker. - Whether the webview resource loader streams a file that size has not been - measured. -- **Where CI gets the build.** `bundle:stats` takes a local build - (`STATS_WEB`). There is no pinned source yet, so `check-vsix.sh` does not - require `stats/` and CI does not run `bundle:stats`. -- **Loading the file from `gemdbStatsHost`** is GemDB Stats' half of the - contract, and is not built yet. Until it is, the panel shows the app with its - own picker. + about 23 s to parse in Chrome. Whether the webview's resource loader streams + it, and whether the app's progress shows, is unmeasured. +- **Reopening after a window reload.** A `WebviewPanel` does not come back + after a reload without a `WebviewPanelSerializer`, and there is none yet. diff --git a/package.json b/package.json index aa46cc6..7208aba 100644 --- a/package.json +++ b/package.json @@ -216,21 +216,6 @@ "when": "gemdb.running && gemdb.hasSavedStacks" } ], - "customEditors": [ - { - "viewType": "gemdb.statistics", - "displayName": "GemDB Stats", - "selector": [ - { - "filenamePattern": "*.out" - }, - { - "filenamePattern": "*.out.gz" - } - ], - "priority": "option" - } - ], "commands": [ { "command": "gemdb.install", @@ -344,6 +329,12 @@ "category": "GemDB", "icon": "$(graph-line)" }, + { + "command": "gemdb.openInStats", + "title": "Open in GemDB Stats", + "category": "GemDB", + "icon": "$(graph-line)" + }, { "command": "gemdb.copyTelemetryId", "title": "Copy Telemetry ID", @@ -592,6 +583,10 @@ { "command": "gemdb.restoreStack", "when": "gemdb.running && gemdb.hasSavedStacks" + }, + { + "command": "gemdb.openInStats", + "when": "false" } ], "debug/variables/context": [ @@ -644,6 +639,13 @@ "when": "debugType == gemdb && !gemdb.savedStackOpen", "group": "z_gemdb@1" } + ], + "explorer/context": [ + { + "command": "gemdb.openInStats", + "when": "resourceFilename =~ /\\.out(\\.gz)?$/", + "group": "navigation@9" + } ] }, "walkthroughs": [ diff --git a/scripts/bundle-stats.mjs b/scripts/bundle-stats.mjs index 8aa9653..662680a 100644 --- a/scripts/bundle-stats.mjs +++ b/scripts/bundle-stats.mjs @@ -7,39 +7,42 @@ // file can be charted wherever the editor is -- including code-server, where // the only screen is a browser tab and a desktop app cannot appear. // -// Like Grail and the MCP server, the payload is a build artifact of another -// repository. Unlike them it is not built here: compiling it needs the Flutter -// SDK, so this script takes a finished `flutter build web` output and turns it -// into something a webview can host. +// Like Grail and the MCP server, the payload is another repository's code, +// pinned in vendor-pins.sh. Unlike them it is not built here: compiling it +// needs the Flutter SDK, so GemDB Stats publishes the web build as a release +// asset, and this script downloads the pinned one and checks its SHA-256. // // Usage: -// STATS_WEB=/path/to/GemDB_Stats/app/build/web npm run bundle:stats +// npm run bundle:stats # the pinned release asset +// STATS_WEB=/path/to/GemDB_Stats/app/build/web npm run bundle:stats # a local build // // Environment: -// STATS_WEB the `flutter build web` output to bundle (required) -// STATS_OUT where to assemble the payload (default: stats/, which is what -// the .vsix ships; the unit tests point it elsewhere) +// STATS_WEB a local `flutter build web` output to bundle instead of downloading +// STATS_URL a web-build .tar.gz to download instead of the pinned one +// STATS_SHA256 its expected SHA-256; unchecked when STATS_URL is given without it +// STATS_OUT where to assemble the payload (default: stats/, which is what +// the .vsix ships; the unit tests point it elsewhere) // -// What it does to the build, and why each step happens HERE rather than when -// the panel opens: everything below can fail, and a failure here is in front -// of whoever is packaging, while the same failure at run time is a blank panel -// on a user's screen. +// What it checks and changes, and why HERE rather than when the panel opens: +// every one of these can fail, and a failure here is in front of whoever is +// packaging, while the same failure at run time is a blank panel on a user's +// screen. GemDB Stats' docs/embedding.md is the other side of this. // +// - The build. It must load CanvasKit from its own files (the page's CSP +// refuses Google's CDN), be dart2js only (the Wasm renderers are left out), +// and start Flutter without a service worker (a webview runs its own). // - The page. A webview needs a Content-Security-Policy, a that -// points at the webview's own resource URL, and a nonce on every script. -// That base is on another origin from the page, so Flutter's history updates -// have to be kept off it (HISTORY_SHIM). And -// Flutter's stock bootstrap registers a service worker, which a webview -// cannot host, so the bootstrap is replaced by a direct call to Flutter's -// loader. The build's own index.html is kept for everything else -- its -// loading screen is GemDB Stats' to design -- and becomes `host.html`, a -// template whose four `{{GEMDB_*}}` placeholders statistics.ts fills. -// - The files. Only the dart2js build with the CanvasKit renderer is kept, -// which is what `flutter build web` without `--wasm` produces. The Wasm -// renderers, debug symbols, the service worker and the PWA manifest go: -// more than half of the build's 47 MB, none of it ever loaded here. +// points at the webview's resource URL, and a nonce on every script. The +// build's index.html becomes `host.html`, a template whose three +// `{{GEMDB_*}}` placeholders statistics.ts fills for each panel; everything +// else on it, the loading screen included, is GemDB Stats' to design. +// - The files. The Wasm renderers, debug symbols, the service worker and the +// PWA manifest never load in a webview: more than half of the build's 47 MB. +import { execFileSync } from 'node:child_process'; +import crypto from 'node:crypto'; import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -72,15 +75,15 @@ function isLeftOut(relative) { const name = path.posix.basename(relative); return ( name.endsWith('.symbols') || - // The Wasm renderers. Only a `--wasm` build loads them. + // The Wasm renderers. Only a `--wasm` build loads them, and checkBuild + // refuses one. /^canvaskit\/(skwasm|wimp)/.test(relative) || // Loaded only when the build config sets `preferWebParagraph`, which - // buildConfigFrom refuses. + // checkBuild refuses. relative.startsWith('canvaskit/webparagraph/') || - // Replaced: host.html takes index.html's place, and calls the loader - // itself instead of running the bootstrap. + // host.html takes its place. relative === 'index.html' || - relative === 'flutter_bootstrap.js' || + // Generated by every build, and registered by none that checkBuild accepts. relative === 'flutter_service_worker.js' || // A PWA's install metadata. A webview cannot install anything. relative === 'manifest.json' || @@ -90,10 +93,11 @@ function isLeftOut(relative) { } /** - * The `_flutter.buildConfig` that `flutter build web` writes into - * flutter_bootstrap.js, reduced to the one build this payload keeps. + * Refuse a build a webview cannot run, judged from flutter_bootstrap.js: the + * `_flutter.buildConfig` that `flutter build web` writes into it, and the + * `_flutter.loader.load(...)` call after it. */ -function buildConfigFrom(bootstrap) { +function checkBuild(bootstrap) { const match = /^_flutter\.buildConfig = (\{.*\});$/m.exec(bootstrap); if (!match) { throw new Error('flutter_bootstrap.js has no `_flutter.buildConfig = {...};` line.'); @@ -112,29 +116,25 @@ function buildConfigFrom(bootstrap) { 'The build chooses its own CanvasKit variant, which may need files this payload leaves out.', ); } - const builds = (config.builds ?? []).filter( - (build) => build.compileTarget === 'dart2js' && build.renderer === 'canvaskit', - ); - if (builds.length !== 1 || builds[0].mainJsPath !== 'main.dart.js') { - throw new Error('The build has no dart2js + CanvasKit entry point at main.dart.js.'); + const builds = config.builds ?? []; + if (!builds.some((b) => b.compileTarget === 'dart2js' && b.mainJsPath === 'main.dart.js')) { + throw new Error('The build has no dart2js entry point at main.dart.js.'); } - return { ...config, builds }; -} - -/** The `\n' + - ' ' - ); + // The loader prefers Wasm where the browser supports it, and its renderers + // are left out of this payload, so a page that offered it would fail there. + if (builds.some((b) => b.compileTarget === 'dart2wasm')) { + throw new Error('The build includes a Wasm build. Build without `--wasm`.'); + } + // Only the code after the config: the flutter.js inlined above it mentions + // service workers in its own implementation. + const start = bootstrap.slice(match.index + match[0].length); + if (/serviceWorker/.test(start)) { + throw new Error( + 'flutter_bootstrap.js registers a service worker, which a webview cannot host. ' + + 'GemDB Stats from PR #10 on starts Flutter without one.', + ); + } + return config; } /** @@ -144,7 +144,9 @@ function loaderScripts(buildConfig) { */ const CSP = [ "default-src 'none'", - // CanvasKit is WebAssembly, which a CSP blocks unless this allows it. + // The bootstrap and what it loads come from the resource origin; the page's + // own inline script carries the nonce. CanvasKit is WebAssembly, which a CSP + // blocks unless 'wasm-unsafe-eval' allows it. "script-src 'nonce-{{GEMDB_NONCE}}' {{GEMDB_CSP_SOURCE}} 'wasm-unsafe-eval'", // Flutter writes its own