[UI updates 2/6] Stage and verify bounded release artifacts

src/updater/artifact.ts

@@ -0,0 +1,191 @@
+import { createHash } from "node:crypto";
+import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { readRelease } from "../cli/releases";
+import { assertCompatible, compatibility } from "./contract";
+import { syncDirectory } from "./journal";
+import { execute } from "./process";
+import {
+  boundedBytes,
+  maxArchiveBytes,
+  type Offer,
+  parseOffer,
+} from "./releases";
+import { inspectData, requireHeadroom, syncTree } from "./snapshot";
+
+export function readSchemaVersions(root: string) {
+  const versions = [];
+  for (const name of ["roost.sqlite", "auth.sqlite"]) {
+    const db = new DatabaseSync(join(root, "data", name), { readOnly: true });
+    try {
+      // Staging runs while the serving app may still hold a write transaction.
+      db.exec("PRAGMA busy_timeout=5000");
+      versions.push(
+        Number(db.prepare("PRAGMA user_version").get()?.user_version),
+      );
+    } finally {
+      db.close();
+    }
+  }
+  return { app: versions[0]!, auth: versions[1]! };
+}
+
+export async function stageArtifact(
+  root: string,
+  offer: Offer,
+  id: string,
+  support: string,
+  signal: AbortSignal,
+  progress: (bytes: number) => void,
+  token?: string,
+  fetcher: typeof fetch = fetch,
+) {
+  const headers = {
+    Accept: "application/vnd.github+json",
+    ...(token ? { Authorization: `Bearer ${token}` } : {}),
+  };
+  const release = await fetcher(
+    `https://api.github.com/repos/${offer.repository}/releases/${offer.releaseId}`,
+    {
+      headers,
+      redirect: "error",
+      signal: AbortSignal.any([signal, AbortSignal.timeout(15000)]),
+    },
+  );
+  if (!release.ok) throw new Error("Pinned release unavailable.");
+  const current = parseOffer(
+    JSON.parse((await boundedBytes(release, 2 * 1024 * 1024)).toString()),
+    offer.repository,
+    offer.checkedAt,
+  );
+  for (const key of [
+    "releaseId",
+    "assetId",
+    "version",
+    "digest",
+    "size",
+  ] as const)
+    if (current[key] !== offer[key])
+      throw new Error("Approved artifact changed.");
+  const data = await inspectData(join(root, "data"));
+  await requireHeadroom(
+    root,
+    data.bytes,
+    data.entries.length,
+    offer.size + 2 * 1024 ** 3,
+    100000,
+  );
+  const directory = join(root, "updates", id);
+  const archive = join(directory, "artifact.tar.gz");
+  const url = `https://api.github.com/repos/${offer.repository}/releases/assets/${offer.assetId}`;
+  let response = await fetcher(url, {
+    headers: { ...headers, Accept: "application/octet-stream" },
+    redirect: "manual",
+    signal: AbortSignal.any([signal, AbortSignal.timeout(300000)]),
+  });
+  if ([301, 302, 303, 307, 308].includes(response.status)) {
+    const target = new URL(response.headers.get("location") ?? "");
+    await response.body?.cancel();
+    if (
+      target.protocol !== "https:" ||
+      target.username ||
+      target.password ||
+      ![
+        "release-assets.githubusercontent.com",
+        "objects.githubusercontent.com",
+      ].includes(target.hostname)
+    )
+      throw new Error("Untrusted artifact redirect.");
+    response = await fetcher(target, {
+      redirect: "error",
+      signal: AbortSignal.any([signal, AbortSignal.timeout(300000)]),
+    });
+  }
+  if (!response.ok || !response.body)
+    throw new Error("Artifact download failed.");
+  const fd = await open(archive, "wx", 0o600);
+  const hash = createHash("sha256");
+  let size = 0;
+  try {
+    for await (const chunk of response.body) {
+      signal.throwIfAborted();
+      size += chunk.length;
+      if (size > offer.size || size > maxArchiveBytes)
+        throw new Error("Artifact size exceeded.");
+      hash.update(chunk);
+      await fd.writeFile(chunk);
+      progress(size);
+    }
+    if (size !== offer.size || `sha256:${hash.digest("hex")}` !== offer.digest)
+      throw new Error("Artifact digest/size mismatch.");
+    await fd.sync();
+  } finally {
+    await fd.close();
+  }
+  signal.throwIfAborted();
+  const unpacked = join(directory, "unpacked");
+  await execute(
+    "/usr/bin/python3",
+    [join(support, "extract.py"), archive, unpacked],
+    300000,
+  );
+  signal.throwIfAborted();
+  const next = await readRelease(unpacked);
+  if (next.version !== offer.version) throw new Error("Manifest/tag mismatch.");
+  const c = compatibility(
+    JSON.parse(await readFile(join(unpacked, "compatibility.json"), "utf8")),
+  );
+  const old = await readRelease(join(root, "current"));
+  const versions = readSchemaVersions(root);
+  assertCompatible(c, {
+    ...versions,
+    codex: old.codex,
+  });
+  if (c.codex !== next.codex)
+    throw new Error("Bundled runtime does not match compatibility contract.");
+  // Rollback must also support the verification gate, not just the candidate.
+  const previousContract = JSON.parse(
+    await readFile(join(root, "current", "compatibility.json"), "utf8"),
+  );
+  assertCompatible(previousContract, {
+    ...versions,
+    codex: old.codex,
+  });
+  if (
+    previousContract.startupGate !== 1 ||
+    JSON.parse(await readFile(join(unpacked, "compatibility.json"), "utf8"))
+      .startupGate !== 1
+  )
+    throw new Error("Both releases require the startup verification gate.");
+  for (const file of ["runtime/node", "runtime/codex/bin/codex"]) {
+    if (!((await lstat(join(unpacked, file))).mode & 0o100))
+      throw new Error("Runtime is not executable.");
+    const output = await execute(join(unpacked, file), ["--version"], 15000);
+    if (
+      output !==
+      (file === "runtime/node" ? `v${next.node}` : `codex-cli ${next.codex}`)
+    )
+      throw new Error("Bundled runtime cannot execute.");
+  }
+  await mkdir(join(root, "releases"), { recursive: true, mode: 0o700 });
+  // A retry may reuse byte-identical verified staging; never replace a retained
+  // release merely because its version name matches a new publisher artifact.
+  try {
+    await lstat(join(root, "releases", offer.version));
+    if (
+      (await inspectData(unpacked)).digest !==
+      (await inspectData(join(root, "releases", offer.version))).digest
+    )
+      throw new Error("Existing candidate differs from the approved artifact.");
+    await rm(unpacked, { recursive: true, force: true });
+    await rm(archive, { force: true });
+    return;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+  }
+  await syncTree(unpacked);
+  await rename(unpacked, join(root, "releases", offer.version));
+  await syncDirectory(join(root, "releases"));
+  await rm(archive, { force: true });
+}

src/updater/extract.py

@@ -0,0 +1,68 @@
+"""Bounded regular-file/directory-only GNU/ustar extractor. No tar subprocess."""
+import gzip, os, pathlib, sys
+archive, destination = sys.argv[1:]
+root = pathlib.Path(destination)
+root.mkdir(mode=0o700)  # fresh owned staging only
+expanded = 0
+seen = set()
+long_name = None
+
+def number(value):
+    text = value.rstrip(b'\0 ').lstrip(b' ')
+    if not text: return 0
+    if any(c not in b'01234567' for c in text): raise ValueError('Invalid tar number')
+    return int(text, 8)
+
+with gzip.open(archive, 'rb') as stream:
+    def read(size):
+        global expanded
+        expanded += size
+        if expanded > 2 * 1024**3: raise ValueError('Expanded archive limit')
+        value = stream.read(size)
+        if len(value) != size: raise ValueError('Truncated archive')
+        return value
+    for count in range(100001):
+        if count == 100000: raise ValueError('Archive entry limit')
+        header = read(512)
+        if header == bytes(512):
+            if read(512) != bytes(512): raise ValueError('Invalid archive end')
+            # Consume bounded padding, rejecting concatenated hidden content.
+            while True:
+                chunk = stream.read(65536)
+                if not chunk: break
+                expanded += len(chunk)
+                if expanded > 2 * 1024**3 or any(chunk): raise ValueError('Trailing archive content')
+            break
+        if sum(header[:148]) + 8 * 32 + sum(header[156:]) != number(header[148:156]): raise ValueError('Tar checksum')
+        size = number(header[124:136])
+        kind = header[156:157]
+        if kind == b'L':
+            if long_name is not None or size > 4096: raise ValueError('Long-name limit')
+            long_name = read(size).rstrip(b'\0').decode('utf-8', 'strict')
+            if size % 512: read(512-size%512)
+            continue
+        if kind not in (b'0', b'\0', b'5'): raise ValueError('Links/extensions/special files forbidden')
+        name = long_name or header[:100].split(b'\0')[0].decode('utf-8','strict')
+        if not long_name and header[257:263] == b'ustar\0':
+            prefix = header[345:500].split(b'\0')[0].decode('utf-8','strict')
+            if prefix: name = prefix + '/' + name
+        long_name = None
+        if len(name)>4096 or '\\' in name or '\0' in name or name.startswith('/') or '..' in name.split('/'): raise ValueError('Unsafe path')
+        name = str(pathlib.PurePosixPath(name))
+        if name in seen: raise ValueError('Duplicate path')
+        seen.add(name)
+        target = root / name
+        if kind == b'5':
+            if size: raise ValueError('Directory payload')
+            target.mkdir(mode=0o700, parents=True, exist_ok=True)
+        else:
+            if name == '.': raise ValueError('Invalid file')
+            target.parent.mkdir(mode=0o700,parents=True,exist_ok=True)
+            with target.open('xb') as output:
+                remaining = size
+                while remaining:
+                    chunk=read(min(65536,remaining)); output.write(chunk); remaining-=len(chunk)
+                output.flush(); os.fsync(output.fileno())
+            os.chmod(target, 0o700 if number(header[100:108]) & 0o111 else 0o600)
+            if size % 512: read(512-size%512)
+    else: raise ValueError('Missing archive end')

tests/update-artifact.test.ts

@@ -0,0 +1,71 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { test } from "node:test";
+import { gzipSync } from "node:zlib";
+
+function archive(name: string, type = "0", contents = "hello") {
+  const header = Buffer.alloc(512);
+  header.write(name, 0, 100, "utf8");
+  header.write("0000644\0", 100);
+  header.write(`${contents.length.toString(8).padStart(11, "0")}\0`, 124);
+  header.fill(32, 148, 156);
+  header.write(type, 156);
+  header.write("ustar\0", 257);
+  const sum = header.reduce((a, b) => a + b, 0);
+  header.write(`${sum.toString(8).padStart(6, "0")}\0 `, 148);
+  return gzipSync(
+    Buffer.concat([
+      header,
+      Buffer.from(contents),
+      Buffer.alloc((512 - (contents.length % 512)) % 512),
+      Buffer.alloc(1024),
+    ]),
+  );
+}
+test("bounded extractor accepts regular GNU/ustar files and refuses path/link/device/extension attacks", async () => {
+  const root = await mkdtemp("/tmp/roost-extraction-");
+  try {
+    const file = join(root, "release.gz");
+    await writeFile(file, archive("./directory/file"));
+    execFileSync(
+      "/usr/bin/python3",
+      ["src/updater/extract.py", file, join(root, "good")],
+      { stdio: "pipe" },
+    );
+    assert.equal(
+      await readFile(join(root, "good/directory/file"), "utf8"),
+      "hello",
+    );
+    let n = 0;
+    for (const [name, type] of [
+      ["../escape", "0"],
+      ["/absolute", "0"],
+      ["link", "2"],
+      ["hardlink", "1"],
+      ["device", "3"],
+      ["pax", "x"],
+      ["file", "6"],
+    ]) {
+      await writeFile(file, archive(name!, type));
+      assert.throws(() =>
+        execFileSync(
+          "/usr/bin/python3",
+          ["src/updater/extract.py", file, join(root, `bad-${n++}`)],
+          { stdio: "pipe" },
+        ),
+      );
+    }
+    await writeFile(file, gzipSync(Buffer.alloc(700)));
+    assert.throws(() =>
+      execFileSync(
+        "/usr/bin/python3",
+        ["src/updater/extract.py", file, join(root, "truncated")],
+        { stdio: "pipe" },
+      ),
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});