From ec5a86562faa5d9580ec2ffd8a0c8e3c87d10711 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 9 Sep 2026 01:27:36 +0000 Subject: [PATCH] Split UI updates 2/6: artifacts --- src/updater/artifact.ts | 191 +++++++++++++++++ src/updater/extract.py | 68 ++++++ tests/update-artifact.test.ts | 71 +++++++ tests/update-download.test.ts | 380 ++++++++++++++++++++++++++++++++++ 4 files changed, 710 insertions(+) create mode 100644 src/updater/artifact.ts create mode 100644 src/updater/extract.py create mode 100644 tests/update-artifact.test.ts create mode 100644 tests/update-download.test.ts diff --git a/src/updater/artifact.ts b/src/updater/artifact.ts new file mode 100644 index 0000000..222747e --- /dev/null +++ b/src/updater/artifact.ts @@ -0,0 +1,191 @@ +import { createHash } from "node:crypto"; +import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { readRelease } from "../cli/releases"; +import { assertCompatible, compatibility } from "./contract"; +import { syncDirectory } from "./journal"; +import { execute } from "./process"; +import { + boundedBytes, + maxArchiveBytes, + type Offer, + parseOffer, +} from "./releases"; +import { inspectData, requireHeadroom, syncTree } from "./snapshot"; + +export function readSchemaVersions(root: string) { + const versions = []; + for (const name of ["roost.sqlite", "auth.sqlite"]) { + const db = new DatabaseSync(join(root, "data", name), { readOnly: true }); + try { + // Staging runs while the serving app may still hold a write transaction. + db.exec("PRAGMA busy_timeout=5000"); + versions.push( + Number(db.prepare("PRAGMA user_version").get()?.user_version), + ); + } finally { + db.close(); + } + } + return { app: versions[0]!, auth: versions[1]! }; +} + +export async function stageArtifact( + root: string, + offer: Offer, + id: string, + support: string, + signal: AbortSignal, + progress: (bytes: number) => void, + token?: string, + fetcher: typeof fetch = fetch, +) { + const headers = { + Accept: "application/vnd.github+json", + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }; + const release = await fetcher( + `https://api.github.com/repos/${offer.repository}/releases/${offer.releaseId}`, + { + headers, + redirect: "error", + signal: AbortSignal.any([signal, AbortSignal.timeout(15000)]), + }, + ); + if (!release.ok) throw new Error("Pinned release unavailable."); + const current = parseOffer( + JSON.parse((await boundedBytes(release, 2 * 1024 * 1024)).toString()), + offer.repository, + offer.checkedAt, + ); + for (const key of [ + "releaseId", + "assetId", + "version", + "digest", + "size", + ] as const) + if (current[key] !== offer[key]) + throw new Error("Approved artifact changed."); + const data = await inspectData(join(root, "data")); + await requireHeadroom( + root, + data.bytes, + data.entries.length, + offer.size + 2 * 1024 ** 3, + 100000, + ); + const directory = join(root, "updates", id); + const archive = join(directory, "artifact.tar.gz"); + const url = `https://api.github.com/repos/${offer.repository}/releases/assets/${offer.assetId}`; + let response = await fetcher(url, { + headers: { ...headers, Accept: "application/octet-stream" }, + redirect: "manual", + signal: AbortSignal.any([signal, AbortSignal.timeout(300000)]), + }); + if ([301, 302, 303, 307, 308].includes(response.status)) { + const target = new URL(response.headers.get("location") ?? ""); + await response.body?.cancel(); + if ( + target.protocol !== "https:" || + target.username || + target.password || + ![ + "release-assets.githubusercontent.com", + "objects.githubusercontent.com", + ].includes(target.hostname) + ) + throw new Error("Untrusted artifact redirect."); + response = await fetcher(target, { + redirect: "error", + signal: AbortSignal.any([signal, AbortSignal.timeout(300000)]), + }); + } + if (!response.ok || !response.body) + throw new Error("Artifact download failed."); + const fd = await open(archive, "wx", 0o600); + const hash = createHash("sha256"); + let size = 0; + try { + for await (const chunk of response.body) { + signal.throwIfAborted(); + size += chunk.length; + if (size > offer.size || size > maxArchiveBytes) + throw new Error("Artifact size exceeded."); + hash.update(chunk); + await fd.writeFile(chunk); + progress(size); + } + if (size !== offer.size || `sha256:${hash.digest("hex")}` !== offer.digest) + throw new Error("Artifact digest/size mismatch."); + await fd.sync(); + } finally { + await fd.close(); + } + signal.throwIfAborted(); + const unpacked = join(directory, "unpacked"); + await execute( + "/usr/bin/python3", + [join(support, "extract.py"), archive, unpacked], + 300000, + ); + signal.throwIfAborted(); + const next = await readRelease(unpacked); + if (next.version !== offer.version) throw new Error("Manifest/tag mismatch."); + const c = compatibility( + JSON.parse(await readFile(join(unpacked, "compatibility.json"), "utf8")), + ); + const old = await readRelease(join(root, "current")); + const versions = readSchemaVersions(root); + assertCompatible(c, { + ...versions, + codex: old.codex, + }); + if (c.codex !== next.codex) + throw new Error("Bundled runtime does not match compatibility contract."); + // Rollback must also support the verification gate, not just the candidate. + const previousContract = JSON.parse( + await readFile(join(root, "current", "compatibility.json"), "utf8"), + ); + assertCompatible(previousContract, { + ...versions, + codex: old.codex, + }); + if ( + previousContract.startupGate !== 1 || + JSON.parse(await readFile(join(unpacked, "compatibility.json"), "utf8")) + .startupGate !== 1 + ) + throw new Error("Both releases require the startup verification gate."); + for (const file of ["runtime/node", "runtime/codex/bin/codex"]) { + if (!((await lstat(join(unpacked, file))).mode & 0o100)) + throw new Error("Runtime is not executable."); + const output = await execute(join(unpacked, file), ["--version"], 15000); + if ( + output !== + (file === "runtime/node" ? `v${next.node}` : `codex-cli ${next.codex}`) + ) + throw new Error("Bundled runtime cannot execute."); + } + await mkdir(join(root, "releases"), { recursive: true, mode: 0o700 }); + // A retry may reuse byte-identical verified staging; never replace a retained + // release merely because its version name matches a new publisher artifact. + try { + await lstat(join(root, "releases", offer.version)); + if ( + (await inspectData(unpacked)).digest !== + (await inspectData(join(root, "releases", offer.version))).digest + ) + throw new Error("Existing candidate differs from the approved artifact."); + await rm(unpacked, { recursive: true, force: true }); + await rm(archive, { force: true }); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + await syncTree(unpacked); + await rename(unpacked, join(root, "releases", offer.version)); + await syncDirectory(join(root, "releases")); + await rm(archive, { force: true }); +} diff --git a/src/updater/extract.py b/src/updater/extract.py new file mode 100644 index 0000000..91a94f5 --- /dev/null +++ b/src/updater/extract.py @@ -0,0 +1,68 @@ +"""Bounded regular-file/directory-only GNU/ustar extractor. No tar subprocess.""" +import gzip, os, pathlib, sys +archive, destination = sys.argv[1:] +root = pathlib.Path(destination) +root.mkdir(mode=0o700) # fresh owned staging only +expanded = 0 +seen = set() +long_name = None + +def number(value): + text = value.rstrip(b'\0 ').lstrip(b' ') + if not text: return 0 + if any(c not in b'01234567' for c in text): raise ValueError('Invalid tar number') + return int(text, 8) + +with gzip.open(archive, 'rb') as stream: + def read(size): + global expanded + expanded += size + if expanded > 2 * 1024**3: raise ValueError('Expanded archive limit') + value = stream.read(size) + if len(value) != size: raise ValueError('Truncated archive') + return value + for count in range(100001): + if count == 100000: raise ValueError('Archive entry limit') + header = read(512) + if header == bytes(512): + if read(512) != bytes(512): raise ValueError('Invalid archive end') + # Consume bounded padding, rejecting concatenated hidden content. + while True: + chunk = stream.read(65536) + if not chunk: break + expanded += len(chunk) + if expanded > 2 * 1024**3 or any(chunk): raise ValueError('Trailing archive content') + break + if sum(header[:148]) + 8 * 32 + sum(header[156:]) != number(header[148:156]): raise ValueError('Tar checksum') + size = number(header[124:136]) + kind = header[156:157] + if kind == b'L': + if long_name is not None or size > 4096: raise ValueError('Long-name limit') + long_name = read(size).rstrip(b'\0').decode('utf-8', 'strict') + if size % 512: read(512-size%512) + continue + if kind not in (b'0', b'\0', b'5'): raise ValueError('Links/extensions/special files forbidden') + name = long_name or header[:100].split(b'\0')[0].decode('utf-8','strict') + if not long_name and header[257:263] == b'ustar\0': + prefix = header[345:500].split(b'\0')[0].decode('utf-8','strict') + if prefix: name = prefix + '/' + name + long_name = None + if len(name)>4096 or '\\' in name or '\0' in name or name.startswith('/') or '..' in name.split('/'): raise ValueError('Unsafe path') + name = str(pathlib.PurePosixPath(name)) + if name in seen: raise ValueError('Duplicate path') + seen.add(name) + target = root / name + if kind == b'5': + if size: raise ValueError('Directory payload') + target.mkdir(mode=0o700, parents=True, exist_ok=True) + else: + if name == '.': raise ValueError('Invalid file') + target.parent.mkdir(mode=0o700,parents=True,exist_ok=True) + with target.open('xb') as output: + remaining = size + while remaining: + chunk=read(min(65536,remaining)); output.write(chunk); remaining-=len(chunk) + output.flush(); os.fsync(output.fileno()) + os.chmod(target, 0o700 if number(header[100:108]) & 0o111 else 0o600) + if size % 512: read(512-size%512) + else: raise ValueError('Missing archive end') diff --git a/tests/update-artifact.test.ts b/tests/update-artifact.test.ts new file mode 100644 index 0000000..bcfdb47 --- /dev/null +++ b/tests/update-artifact.test.ts @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { test } from "node:test"; +import { gzipSync } from "node:zlib"; + +function archive(name: string, type = "0", contents = "hello") { + const header = Buffer.alloc(512); + header.write(name, 0, 100, "utf8"); + header.write("0000644\0", 100); + header.write(`${contents.length.toString(8).padStart(11, "0")}\0`, 124); + header.fill(32, 148, 156); + header.write(type, 156); + header.write("ustar\0", 257); + const sum = header.reduce((a, b) => a + b, 0); + header.write(`${sum.toString(8).padStart(6, "0")}\0 `, 148); + return gzipSync( + Buffer.concat([ + header, + Buffer.from(contents), + Buffer.alloc((512 - (contents.length % 512)) % 512), + Buffer.alloc(1024), + ]), + ); +} +test("bounded extractor accepts regular GNU/ustar files and refuses path/link/device/extension attacks", async () => { + const root = await mkdtemp("/tmp/roost-extraction-"); + try { + const file = join(root, "release.gz"); + await writeFile(file, archive("./directory/file")); + execFileSync( + "/usr/bin/python3", + ["src/updater/extract.py", file, join(root, "good")], + { stdio: "pipe" }, + ); + assert.equal( + await readFile(join(root, "good/directory/file"), "utf8"), + "hello", + ); + let n = 0; + for (const [name, type] of [ + ["../escape", "0"], + ["/absolute", "0"], + ["link", "2"], + ["hardlink", "1"], + ["device", "3"], + ["pax", "x"], + ["file", "6"], + ]) { + await writeFile(file, archive(name!, type)); + assert.throws(() => + execFileSync( + "/usr/bin/python3", + ["src/updater/extract.py", file, join(root, `bad-${n++}`)], + { stdio: "pipe" }, + ), + ); + } + await writeFile(file, gzipSync(Buffer.alloc(700))); + assert.throws(() => + execFileSync( + "/usr/bin/python3", + ["src/updater/extract.py", file, join(root, "truncated")], + { stdio: "pipe" }, + ), + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/tests/update-download.test.ts b/tests/update-download.test.ts new file mode 100644 index 0000000..22147db --- /dev/null +++ b/tests/update-download.test.ts @@ -0,0 +1,380 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { + chmod, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { test } from "node:test"; +import { activate } from "../src/cli/releases"; +import { readSchemaVersions, stageArtifact } from "../src/updater/artifact"; +import { parseOffer } from "../src/updater/releases"; + +test("staging schema reads wait for transient writers and bound persistent contention", async () => { + const root = await mkdtemp("/tmp/roost-schema-lock-"); + try { + await mkdir(join(root, "data")); + for (const name of ["roost.sqlite", "auth.sqlite"]) { + const db = new DatabaseSync(join(root, "data", name)); + db.exec(`PRAGMA user_version=${name === "roost.sqlite" ? 10 : 0}`); + db.close(); + } + for (const [name, transient] of [ + ["roost.sqlite", true], + ["auth.sqlite", true], + ["auth.sqlite", false], + ] as const) { + const writer = spawn( + process.execPath, + [ + "--input-type=module", + "-e", + `import {DatabaseSync} from 'node:sqlite'; + const db=new DatabaseSync(process.argv[1]); + db.exec('BEGIN EXCLUSIVE'); + process.stdout.write('locked\\n'); + process.stdin.once('data',()=>setTimeout(()=>{db.exec('COMMIT');db.close();process.exit(0)},200));`, + join(root, "data", name), + ], + { stdio: ["pipe", "pipe", "ignore"] }, + ); + const exited = once(writer, "exit"); + try { + assert.equal( + String((await once(writer.stdout!, "data"))[0]), + "locked\n", + ); + if (transient) writer.stdin!.write("release\n"); + const start = Date.now(); + if (transient) { + assert.deepEqual(readSchemaVersions(root), { app: 10, auth: 0 }); + assert.ok( + Date.now() - start >= 100, + "The actual writer lock was reached", + ); + } else { + assert.throws(() => readSchemaVersions(root), /database is locked/); + assert.ok(Date.now() - start >= 4500); + assert.ok(Date.now() - start < 10000); + } + } finally { + if (!transient && writer.exitCode === null) + writer.stdin!.end("release\n"); + await exited; + } + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("pinned download validates identity/digest/manifest and strips credentials on trusted redirects", async () => { + const root = await mkdtemp("/tmp/roost-download-"); + const compatibility = { + protocol: 1, + startupGate: 1, + app: { min: 10, max: 10, output: 10 }, + auth: { min: 0, max: 0, output: 0 }, + data: "complete-snapshot-v1", + externalState: "unchanged", + codex: "0.153.4", + }; + const manifest = { + schema: 1, + version: "0.1.40", + platform: "linux", + arch: "x64", + node: "24.15.0", + codex: "0.153.4", + }; + try { + await mkdir(join(root, "data")); + for (const name of ["roost.sqlite", "auth.sqlite"]) { + const db = new DatabaseSync(join(root, "data", name)); + db.exec(`PRAGMA user_version=${name === "roost.sqlite" ? 10 : 0}`); + db.close(); + } + for (const version of ["0.1.40", "0.1.41"]) { + const target = join( + root, + version === "0.1.40" ? "releases" : "fixture", + version, + ); + for (const [path, contents] of Object.entries({ + "release.json": JSON.stringify({ ...manifest, version }), + "compatibility.json": JSON.stringify(compatibility), + "runtime/node": "#!/bin/sh\necho v24.15.0\n", + "runtime/codex/bin/codex": "#!/bin/sh\necho codex-cli 0.153.4\n", + "cli/roost.mjs": "export {};", + "app/server/index.mjs": "export {};", + "bin/roost": "#!/bin/sh\n", + })) { + await mkdir(join(target, path, ".."), { recursive: true }); + await writeFile(join(target, path), contents); + await chmod(join(target, path), 0o700); + } + } + await activate(root, join(root, "releases", "0.1.40")); + const archive = join(root, "fixture.tar.gz"); + execFileSync("tar", [ + "--format=gnu", + "-czf", + archive, + "-C", + join(root, "fixture", "0.1.41"), + ".", + ]); + const bytes = await readFile(archive); + const metadata = { + id: 1, + tag_name: "v0.1.41", + draft: false, + prerelease: false, + assets: [ + { + id: 2, + name: "roost-linux-x64.tar.gz", + size: bytes.length, + digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`, + url: "https://api.github.com/repos/srctl/roost/releases/assets/2", + }, + ], + }; + const offer = parseOffer(metadata, "srctl/roost", Date.now()); + for (const mode of [ + "changed", + "evil-redirect", + "bad-digest", + "success", + "retry", + ] as const) { + let calls = 0; + const fetcher = (async (url, init) => { + calls++; + if (calls === 1) + return Response.json( + mode === "changed" + ? { + ...metadata, + assets: [ + { + ...metadata.assets[0], + id: 3, + url: "https://api.github.com/repos/srctl/roost/releases/assets/3", + }, + ], + } + : metadata, + ); + if (calls === 2) { + assert.equal( + new Headers(init?.headers).get("authorization"), + "Bearer private-token", + ); + return new Response(null, { + status: 302, + headers: { + location: + mode === "evil-redirect" + ? "https://evil.example/payload" + : "https://release-assets.githubusercontent.com/payload", + }, + }); + } + assert.equal( + String(url), + "https://release-assets.githubusercontent.com/payload", + ); + assert.equal(new Headers(init?.headers).get("authorization"), null); + return new Response( + mode === "bad-digest" ? Buffer.alloc(bytes.length) : bytes, + ); + }) as typeof fetch; + const id = randomUUID(); + await mkdir(join(root, "updates", id), { recursive: true }); + const run = stageArtifact( + root, + offer, + id, + resolve("src/updater"), + new AbortController().signal, + () => {}, + "private-token", + fetcher, + ); + if (["success", "retry"].includes(mode)) await run; + else await assert.rejects(run); + assert.equal( + await readFile(join(root, "current", "release.json"), "utf8"), + JSON.stringify(manifest), + ); + } + const fixture = join(root, "fixture", "0.1.41"); + const validManifest = { ...manifest, version: "0.1.41" }; + for (const [name, file, contents, error] of [ + [ + "tag", + "release.json", + JSON.stringify({ ...validManifest, version: "0.1.42" }), + /Manifest\/tag mismatch/, + ], + [ + "platform", + "release.json", + JSON.stringify({ ...validManifest, arch: "arm64" }), + /invalid Roost release manifest/, + ], + [ + "protocol", + "compatibility.json", + JSON.stringify({ ...compatibility, protocol: 2 }), + /compatibility contract/, + ], + [ + "app schema", + "compatibility.json", + JSON.stringify({ + ...compatibility, + app: { min: 11, max: 11, output: 11 }, + }), + /Database or bundled Codex compatibility/, + ], + [ + "auth schema", + "compatibility.json", + JSON.stringify({ + ...compatibility, + auth: { min: 1, max: 1, output: 1 }, + }), + /Database or bundled Codex compatibility/, + ], + [ + "Codex", + "compatibility.json", + JSON.stringify({ ...compatibility, codex: "0.154.0" }), + /Database or bundled Codex compatibility/, + ], + [ + "startup gate", + "compatibility.json", + JSON.stringify({ ...compatibility, startupGate: 0 }), + /startup verification gate/, + ], + [ + "runtime version substring", + "runtime/node", + "#!/bin/sh\necho v124.15.0\n", + /Bundled runtime cannot execute/, + ], + ["runtime ABI", "runtime/node", "#!/unavailable-abi-interpreter\n", /./], + ] as const) { + const original = await readFile(join(fixture, file)); + try { + await writeFile(join(fixture, file), contents); + execFileSync("tar", [ + "--format=gnu", + "-czf", + archive, + "-C", + fixture, + ".", + ]); + const payload = await readFile(archive); + const release = { + ...metadata, + assets: [ + { + ...metadata.assets[0]!, + size: payload.length, + digest: `sha256:${createHash("sha256").update(payload).digest("hex")}`, + }, + ], + }; + const invalid = parseOffer(release, "srctl/roost", Date.now()); + const id = randomUUID(); + await mkdir(join(root, "updates", id), { recursive: true }); + let calls = 0; + const fetcher = (async () => + ++calls === 1 + ? Response.json(release) + : new Response(payload)) as typeof fetch; + await assert.rejects( + stageArtifact( + root, + invalid, + id, + resolve("src/updater"), + new AbortController().signal, + () => {}, + undefined, + fetcher, + ), + error, + name, + ); + assert.equal( + calls, + 2, + `${name}: artifact must reach actual validation`, + ); + assert.equal( + await readFile(join(root, "current", "release.json"), "utf8"), + JSON.stringify(manifest), + ); + } finally { + await writeFile(join(fixture, file), original); + } + } + for (const prior of [ + { ...compatibility, app: { min: 0, max: 9, output: 9 } }, + { ...compatibility, codex: "0.154.0" }, + ]) { + await writeFile( + join(root, "current", "compatibility.json"), + JSON.stringify(prior), + ); + const id = randomUUID(); + await mkdir(join(root, "updates", id), { recursive: true }); + let calls = 0; + const fetcher = (async () => + ++calls === 1 + ? Response.json(metadata) + : new Response(bytes)) as typeof fetch; + await assert.rejects( + stageArtifact( + root, + offer, + id, + resolve("src/updater"), + new AbortController().signal, + () => {}, + undefined, + fetcher, + ), + /Database or bundled Codex compatibility/, + ); + } + await writeFile( + join(root, "current", "compatibility.json"), + JSON.stringify(compatibility), + ); + assert.equal( + JSON.parse( + await readFile( + join(root, "releases", "0.1.41", "release.json"), + "utf8", + ), + ).version, + "0.1.41", + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +});