From 94b49cd169fe830d65a4974b7a98240507eb2f6f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 20:37:58 +0000 Subject: [PATCH] feat(cli): replace the cartesi-machine spawn with @cartesi/machine Configure, boot, store and hash the Cartesi machine through the @cartesi/machine N-API bindings, instead of spawning cartesi-machine and cartesi-machine-stored-hash (falling back to running them inside the SDK docker image). machine.ts now translates a cartesi.toml Config into an emulator MachineConfig directly, mirroring what the cartesi-machine CLI does with its command line: the boot args it appends to, the init script (splash, flash drive mounts and chowns, nvram permissions and chowns, environment exports, WORKDIR and USER), the flash drives with root first so it lands on pmem0, the nvram ranges in declaration order, and the virtio console setup for an interactive shell. buildMachineConfig is a pure function, covered by unit tests. With no SDK image to take the kernel from, images.ts downloads the pinned cartesi/machine-linux-image release on first use, verifies its checksum and caches it under XDG_CACHE_HOME. CARTESI_IMAGES_PATH and machine.ram_image still win. The run loop can capture the guest console into a file and return it, which is how a test reads what the machine printed now that there is no child process to read stdout from. The emulator moves from 0.20 to 0.21, so machine hashes change. The standalone binaries are gone: a bun single file executable has no node_modules, and the addon resolves its platform .node at runtime, so it cannot be embedded. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01UTEd5g3mF849BATTstssR3 --- .changeset/olive-donkeys-shave.md | 29 ++ .github/workflows/release.yaml | 11 - CLAUDE.md | 6 +- apps/cli/build.ts | 33 +- apps/cli/package.json | 1 + apps/cli/src/base.ts | 29 +- apps/cli/src/commands/build.ts | 22 +- apps/cli/src/commands/doctor.ts | 13 +- apps/cli/src/commands/run.ts | 9 +- apps/cli/src/commands/shell.ts | 31 +- apps/cli/src/config.ts | 4 +- .../src/exec/cartesi-machine-stored-hash.ts | 44 +- apps/cli/src/exec/cartesi-machine.ts | 180 +++++-- apps/cli/src/images.ts | 156 +++++++ apps/cli/src/machine.ts | 442 +++++++++++++----- .../exec/cartesi-machine-stored-hash.test.ts | 50 +- .../integration/exec/cartesi-machine.test.ts | 27 -- .../tests/integration/machine/nvram.test.ts | 24 +- apps/cli/tests/unit/config/fixtures/full.toml | 2 +- .../tests/unit/exec/cartesi-machine.test.ts | 45 +- apps/cli/tests/unit/machine.test.ts | 405 ++++++++++++++-- bun.lock | 15 + 22 files changed, 1164 insertions(+), 414 deletions(-) create mode 100644 .changeset/olive-donkeys-shave.md create mode 100644 apps/cli/src/images.ts delete mode 100644 apps/cli/tests/integration/exec/cartesi-machine.test.ts diff --git a/.changeset/olive-donkeys-shave.md b/.changeset/olive-donkeys-shave.md new file mode 100644 index 00000000..fe6a5925 --- /dev/null +++ b/.changeset/olive-donkeys-shave.md @@ -0,0 +1,29 @@ +--- +"@cartesi/cli": minor +--- + +Replace the `cartesi-machine` subprocess with the `@cartesi/machine` bindings + +The Cartesi machine is now configured, booted, stored and hashed through +[`@cartesi/machine`](https://github.com/cartesi/rollups-ts), an N-API addon, so `build`, `shell` +and `status` no longer shell out to `cartesi-machine` or `cartesi-machine-stored-hash`, and no +longer fall back to running them inside the SDK Docker image. + +Notable consequences: + +- **The machine emulator moved from 0.20 to 0.21**, which is the version `@cartesi/machine` + links against. Machine hashes change, and applications have to be redeployed. +- **The Linux kernel image is downloaded and cached.** With no SDK image to take it from, the + default `ram_image` now comes from the pinned `cartesi/machine-linux-image` v0.21.0 release, + fetched on first use into `$XDG_CACHE_HOME/cartesi/images` (`~/.cache/cartesi/images`) and + verified against its SHA-256. A `CARTESI_IMAGES_PATH` directory containing the image is used + when set, and `machine.ram_image` in `cartesi.toml` still takes precedence over both. +- **A snapshot is read by the linked emulator.** `cartesi hash` and `cartesi status` no longer + run `cartesi-machine-stored-hash` in the project's SDK image, so a snapshot stored by an + incompatible emulator reads as no hash and has to be rebuilt. +- **Boot args are no longer double quoted.** The old code passed `--append-bootargs=""` to + the CLI without a shell, so the quotes ended up in the kernel command line. They are gone now. +- **The standalone binaries are no longer built or released.** A Bun single file executable has + no `node_modules`, and the addon resolves its platform specific `.node` at runtime, so it + cannot be embedded — not even for the host platform. npm is the only distribution now, and the + homebrew formula has to install the package from there. diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8a3ea37e..efc6deec 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -70,17 +70,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Release CLI binaries - if: ${{ steps.changeset.outputs.published == 'true' && contains(fromJSON(steps.changeset.outputs.publishedPackages).*.name, '@cartesi/cli') }} - run: | - for f in cartesi-*; do tar -czf "$f.tar.gz" "$f"; done - VERSION=$(jq -r '.[] | select(.name=="@cartesi/cli") | .version' <<< '${{ steps.changeset.outputs.publishedPackages }}') - TAG="@cartesi/cli@${VERSION}" - gh release upload "$TAG" cartesi-*.tar.gz - working-directory: ./apps/cli/bin - env: - GH_TOKEN: ${{ github.token }} - build_sdk: name: Build SDK needs: release diff --git a/CLAUDE.md b/CLAUDE.md index 986d1553..520e6c4f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,7 +40,7 @@ bun test apps/cli/tests/unit/config.test.ts # Run a single test bun run build --filter @cartesi/devnet ``` -The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). It also produces native binaries for darwin-arm64, darwin-x64, linux-arm64, linux-x64 in `apps/cli/bin/`. +The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). `@cartesi/machine` is left external — it is a native addon that resolves its platform binary at runtime and cannot be bundled, which is also why there are no standalone `bun --compile` binaries. ## Architecture @@ -55,7 +55,9 @@ The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation - **`commands/`** — Each file exports a `create*Command()` function returning a Commander command. Main commands: `build`, `run`, `deploy`, `send`, `deposit`, `create`, `doctor`, `shell`, `clean`, `hash`, `logs`, `status`, `address-book`. - **`builder/`** — Drive builder implementations (directory, docker, tar, empty, none). Each builder produces ext2 or SquashFS filesystems for Cartesi Machine drives. - **`compose/`** — Docker Compose service definitions generated as TypeScript objects (anvil, node, bundler, database, paymaster, proxy, explorer, etc.). -- **`exec/`** — Wrappers around subprocess execution (cartesi-machine, rollups) using `execa`. +- **`exec/`** — Machine and filesystem tooling. `cartesi-machine` and `cartesi-machine-stored-hash` are native N-API bindings (`@cartesi/machine`); `genext2fs`, `mksquashfs` and `rollups` still spawn subprocesses via `execa`, falling back to `docker run` against the SDK image. +- **`machine.ts`** — Translates a `cartesi.toml` `Config` into an emulator `MachineConfig` (bootargs, `dtb.init`, flash drives, nvrams), mirroring what the `cartesi-machine` CLI does with its command line. +- **`images.ts`** — Downloads and caches the Linux kernel image the machine boots, from a pinned `cartesi/machine-linux-image` release. - **`config.ts`** — Parses `cartesi.toml` (TOML-based project config) into typed `Config` objects. Defines drive configs, machine configs, and SDK versions. - **`contracts.ts`** — Generated contract addresses and ABI bindings (via `@wagmi/cli`). - **`wallet.ts`** — Wallet utilities using `viem` for Ethereum interaction. diff --git a/apps/cli/build.ts b/apps/cli/build.ts index 3ed24ffb..2bd8b21c 100644 --- a/apps/cli/build.ts +++ b/apps/cli/build.ts @@ -1,35 +1,22 @@ +// the emulator binding resolves its platform binary at runtime, so it can never +// be bundled: it is left as an import, resolved from node_modules +const external = ["@cartesi/machine"]; + // build for npm package await Bun.build({ banner: "#!/usr/bin/env node", entrypoints: ["./src/index.ts"], + external, minify: true, outdir: "dist", sourcemap: true, target: "node", }); -// build bun binaries for all supported platforms -const targets: Bun.Build.CompileTarget[] = [ - "bun-darwin-arm64", - "bun-darwin-x64", - "bun-linux-arm64", - "bun-linux-x64", -]; - -await Promise.all( - targets.map((target) => - Bun.build({ - bytecode: true, - compile: { - outfile: `bin/cartesi-${target.replace("bun-", "")}`, - target, - }, - entrypoints: ["./src/index.ts"], - minify: true, - sourcemap: "linked", - target: "bun", - }), - ), -); +// NOTE: the standalone binaries this used to cross-compile (bin/cartesi-*) +// are gone. A single file executable has no node_modules, and the emulator +// binding resolves its platform specific .node at runtime, so it cannot be +// embedded — not even for the host platform. The npm package is the only +// distribution now, and the homebrew formula has to install it from there. export {}; diff --git a/apps/cli/package.json b/apps/cli/package.json index 8ef34934..e9befd08 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -15,6 +15,7 @@ "/dist" ], "dependencies": { + "@cartesi/machine": "^1.0.0-alpha.2", "@commander-js/extra-typings": "^15.0.0", "@inquirer/confirm": "^6.3.3", "@inquirer/core": "^12.0.4", diff --git a/apps/cli/src/base.ts b/apps/cli/src/base.ts index 4d805326..199e7bd7 100644 --- a/apps/cli/src/base.ts +++ b/apps/cli/src/base.ts @@ -41,33 +41,14 @@ export const getContextPath = (...paths: string[]): string => { }; /** - * SDK image of the project, which built its machine snapshot + * Read the hash of the cartesi machine snapshot, if one exists. The snapshot is + * loaded by the emulator the bindings link against, so one stored by an + * incompatible emulator reads as undefined and has to be rebuilt. */ -const getProjectSdk = (): string | undefined => { - try { - return getApplicationConfig(["cartesi.toml"]).sdk; - } catch { - // an invalid config is reported by the commands that build with it - return undefined; - } -}; - -/** - * Read the hash of the cartesi machine snapshot, if one exists. Without a - * local cartesi-machine-stored-hash, it runs in the SDK image the snapshot was - * built with, as an emulator of another version may not load it. - * @param options sdk image of the project, read from cartesi.toml if not given - */ -export const getMachineHash = async (options?: { - sdk?: string; -}): Promise => { +export const getMachineHash = async (): Promise => { const imagePath = getContextPath("image"); if (fs.existsSync(imagePath)) { - const image = options?.sdk ?? getProjectSdk(); - return await cartesiMachineStoredHash.computeHash( - imagePath, - image ? { image } : undefined, - ); + return await cartesiMachineStoredHash.computeHash(imagePath); } return undefined; }; diff --git a/apps/cli/src/commands/build.ts b/apps/cli/src/commands/build.ts index e805da9d..edb12302 100755 --- a/apps/cli/src/commands/build.ts +++ b/apps/cli/src/commands/build.ts @@ -188,20 +188,30 @@ export const createBuildCommand = () => { } // create machine snapshot - await bootMachine( + const { exitCode, rootHash } = await bootMachine( config, result.imageInfo, { + cwd: destination, finalHash: true, + reporter: (line) => console.error(line), store: "image", }, - { - cwd: destination, - stdio: "inherit", - }, ); - // make snapshot readable by all users, because cartesi-machine sets to 600 + if (exitCode !== 0) { + throw new Error( + exitCode === 2 + ? "Machine did not stop at a rollup accept, it is not a valid rolling template" + : `Machine stopped with exit code ${exitCode}`, + ); + } + + if (rootHash) { + console.error(`Machine hash: ${chalk.cyan(rootHash)}`); + } + + // make snapshot readable by all users, because the emulator sets to 600 await fs.chmod(path.join(destination, "image"), 0o755); }); }; diff --git a/apps/cli/src/commands/doctor.ts b/apps/cli/src/commands/doctor.ts index c54c1d72..920cde63 100755 --- a/apps/cli/src/commands/doctor.ts +++ b/apps/cli/src/commands/doctor.ts @@ -3,7 +3,6 @@ import chalk from "chalk"; import { execa } from "execa"; import ora, { type Ora } from "ora"; import semver from "semver"; -import { DEFAULT_SDK_IMAGE, DEFAULT_SDK_VERSION } from "../config.js"; import { cartesiMachine } from "../exec/index.js"; const MINIMUM_DOCKER_VERSION = "25.0.0"; // Replace with our minimum required Docker version @@ -124,16 +123,12 @@ const checkBuildx = async (progress: Ora): Promise => { const checkCartesiMachine = async (progress: Ora): Promise => { progress.start("Checking Cartesi Machine version..."); - // doctor does not read cartesi.toml, so check against the default sdk image. the host binary - // still takes precedence, which is the install most likely to be out of date - const v = await cartesiMachine.version({ - image: `${DEFAULT_SDK_IMAGE}:${DEFAULT_SDK_VERSION}`, - }); + // the bindings link against the emulator, so this reports the version + // compiled into the CLI rather than probing an install + const v = cartesiMachine.version(); if (v === null) { - throw new Error( - "Could not determine the Cartesi Machine version. Check that Docker is running.", - ); + throw new Error("Could not determine the Cartesi Machine version."); } if (!semver.satisfies(v.format(), cartesiMachine.requiredVersion)) { throw new Error( diff --git a/apps/cli/src/commands/run.ts b/apps/cli/src/commands/run.ts index bc84b331..4f29d554 100755 --- a/apps/cli/src/commands/run.ts +++ b/apps/cli/src/commands/run.ts @@ -83,7 +83,6 @@ const shell = async (options: { projectName: string; prt?: boolean; salt: number; - sdk: string; withdrawalConfig?: WithdrawalConfig; claimStagingPeriod: number; }) => { @@ -93,7 +92,6 @@ const shell = async (options: { log, projectName, prt, - sdk, withdrawalConfig, claimStagingPeriod, } = options; @@ -176,7 +174,7 @@ const shell = async (options: { await build?.parseAsync([], { from: "user" }); // redeploy - const hash = await getMachineHash({ sdk }); + const hash = await getMachineHash(); if (hash) { if (lastDeployment) { await undeploy({ projectName }); @@ -495,9 +493,7 @@ export const createRunCommand = () => { // deploy the application let deployment: RollupsDeployment | undefined; let salt = 0; - const hash = await getMachineHash({ - sdk: applicationConfig.sdk, - }); + const hash = await getMachineHash(); if (hash) { deployment = await deploy({ epochLength, @@ -548,7 +544,6 @@ export const createRunCommand = () => { projectName, prt, salt, - sdk: applicationConfig.sdk, claimStagingPeriod, withdrawalConfig: applicationConfig?.withdrawalConfig, }); diff --git a/apps/cli/src/commands/shell.ts b/apps/cli/src/commands/shell.ts index fb071a86..ed9280d9 100755 --- a/apps/cli/src/commands/shell.ts +++ b/apps/cli/src/commands/shell.ts @@ -1,5 +1,4 @@ import { Command } from "@commander-js/extra-typings"; -import { ExecaError } from "execa"; import fs from "fs-extra"; import path from "node:path"; import { getApplicationConfig, getContextPath } from "../base.js"; @@ -51,26 +50,16 @@ export const createShellCommand = () => { // run as root if flag is set config.machine.user = runAsRoot ? "root" : undefined; - // boot machine - try { - await bootMachine( - config, - undefined, - { interactive: true }, // start with interactive mode on - { - cwd: destination, - stdio: "inherit", - tty: true, - }, - ); - } catch (error: unknown) { - if (error instanceof ExecaError) { - // just continue gracefully - if (error.exitCode === 130) { - return; - } - throw error; - } + // boot machine, in interactive mode + const { exitCode } = await bootMachine(config, undefined, { + cwd: destination, + interactive: true, + reporter: (line) => console.error(line), + }); + + // 130 is the shell being interrupted, which is not a failure + if (exitCode !== 0 && exitCode !== 130) { + throw new Error(`Machine stopped with exit code ${exitCode}`); } }); }; diff --git a/apps/cli/src/config.ts b/apps/cli/src/config.ts index 82ce77e0..12a9b096 100644 --- a/apps/cli/src/config.ts +++ b/apps/cli/src/config.ts @@ -219,9 +219,9 @@ export type MachineConfig = { entrypoint?: string; env: Record; // explicit environment variables injected into cartesi-machine ENV envFile?: string; // path to a .env file with environment variables injected into cartesi-machine ENV - maxMCycle?: bigint; // default given by cartesi-machine + maxMCycle?: bigint; // default is no limit ramLength: string; - ramImage?: string; // default given by cartesi-machine + ramImage?: string; // default is the pinned cartesi machine-linux-image release useDockerEnv: boolean; // inject docker image ENV into cartesi-machine ENV useDockerWorkdir: boolean; // inject docker image WORKDIR into cartesi-machine WORKDIR user?: string; // default given by cartesi-machine diff --git a/apps/cli/src/exec/cartesi-machine-stored-hash.ts b/apps/cli/src/exec/cartesi-machine-stored-hash.ts index 004affd9..85e15eb3 100644 --- a/apps/cli/src/exec/cartesi-machine-stored-hash.ts +++ b/apps/cli/src/exec/cartesi-machine-stored-hash.ts @@ -1,45 +1,21 @@ -import { isHash, type Hash } from "viem"; -import { DEFAULT_SDK_IMAGE, DEFAULT_SDK_VERSION } from "../config.js"; -import { execaDockerFallback, type DockerFallbackOptions } from "./util.js"; - -type ComputeHashOptions = { cwd?: string } & DockerFallbackOptions; +import { load } from "@cartesi/machine"; +import { bytesToHex, type Hash } from "viem"; /** - * - * @param machineDir - * @param options - * @returns + * Reads the root hash of a stored Cartesi machine snapshot. + * @param machineDir directory holding the machine snapshot + * @returns the machine hash, or undefined if the snapshot can't be read */ export const computeHash = async ( machineDir: string, - options?: ComputeHashOptions, ): Promise => { - const defaultImage = `${DEFAULT_SDK_IMAGE}:${DEFAULT_SDK_VERSION}`; - const execaOptions = Object.assign( - {}, - { image: defaultImage, cwd: process.cwd() }, - options, - ); - try { - const { stdout } = await execaDockerFallback( - "cartesi-machine-stored-hash", - [machineDir], - execaOptions, - ); - - if (undefined !== stdout) { - // cartesi-machine-stored-hash prints a bare digest up to emulator - // 0.20 and a 0x-prefixed one from 0.21 on. - const digest = stdout.toString().trim(); - const hash = digest.startsWith("0x") ? digest : `0x${digest}`; - - if (isHash(hash)) { - return hash; - } + const machine = load(machineDir); + try { + return bytesToHex(machine.getRootHash()) as Hash; + } finally { + machine.destroy(); } - - return undefined; } catch { return undefined; } diff --git a/apps/cli/src/exec/cartesi-machine.ts b/apps/cli/src/exec/cartesi-machine.ts index 30571500..5bcb6a93 100644 --- a/apps/cli/src/exec/cartesi-machine.ts +++ b/apps/cli/src/exec/cartesi-machine.ts @@ -1,36 +1,160 @@ -import { parse, Range, satisfies, type SemVer } from "semver"; import { - execaDockerFallback, - type ExecaOptionsDockerFallback, -} from "./util.js"; + BreakReason, + create, + getDefaultConfig, + getVersion, + HtifYieldCommand, + HtifYieldReason, + type MachineConfig, + type MachineRuntimeConfig, + MAX_MCYCLE, + Reg, +} from "@cartesi/machine"; +import { parse, Range, satisfies, type SemVer } from "semver"; +import fs from "node:fs"; +import tmp from "tmp"; +import { bytesToHex, type Hash } from "viem"; export const requiredVersion = new Range("^0.21.0"); -export const boot = ( - args: readonly string[], - options: ExecaOptionsDockerFallback, -) => execaDockerFallback("cartesi-machine", args, options); +export type RunOptions = { + /** fail unless the machine stopped at a rollup accept yield */ + assertRollingTemplate?: boolean; + /** + * collect what the guest writes to its console into `stdout`, instead of + * letting it through to the terminal + */ + captureOutput?: boolean; + /** compute the machine root hash once the run is over */ + finalHash?: boolean; + /** target mcycle to stop at, defaults to no limit */ + maxMCycle?: bigint; + /** directory to store the machine snapshot into */ + store?: string; +}; + +export type RunResult = { + breakReason: BreakReason; + /** exit code of the guest, mirroring what the cartesi-machine CLI reports */ + exitCode: number; + rootHash?: Hash; + /** console output of the guest, when `captureOutput` asked for it */ + stdout?: string; +}; + +/** Machine configuration defaults, as filled in by the emulator itself. */ +export const defaultConfig = (): MachineConfig => getDefaultConfig(); + +/** + * A machine stopped at a halt, a manual yield, or an mcycle overflow no longer + * advances on its own. + */ +const isAtFixedPoint = (breakReason: BreakReason): boolean => + breakReason === BreakReason.Halted || + breakReason === BreakReason.YieldedManually || + breakReason === BreakReason.McycleOverflow; -export const version = async ( - options?: ExecaOptionsDockerFallback, -): Promise => { +/** + * Creates a machine, runs it to a fixed point (or to the requested mcycle), + * and optionally hashes and stores it. Automatic yields are acknowledged and + * discarded, and console I/O breaks just resume the run, which is what the + * cartesi-machine CLI does for a plain boot. + */ +export const run = ( + config: MachineConfig, + runtimeConfig: MachineRuntimeConfig | undefined, + options: RunOptions = {}, +): RunResult => { + // the emulator has no API to read a captured console back, so it writes to + // a file that is read once the run is over. flushing every character keeps + // the file complete without depending on when the machine is torn down + const outputFile = options.captureOutput ? tmp.fileSync() : undefined; + if (outputFile) { + runtimeConfig = { + ...runtimeConfig, + console: { + ...runtimeConfig?.console, + output_destination: "to_file", + output_filename: outputFile.name, + output_flush_mode: "every_char", + }, + }; + } + + const machine = create(config, runtimeConfig); try { - const { stdout } = await execaDockerFallback( - "cartesi-machine", - ["--version-json"], - // cwd is interpolated into the docker volume mount, so it must be defined - { ...options, cwd: options?.cwd ?? process.cwd() }, - ); - if (typeof stdout === "string") { - const output = JSON.parse(stdout); - return parse(output.version); + const target = options.maxMCycle ?? MAX_MCYCLE; + let breakReason: BreakReason; + for (;;) { + breakReason = machine.run(target); + if ( + isAtFixedPoint(breakReason) || + breakReason === BreakReason.ReachedTargetMcycle + ) { + break; + } + if (breakReason === BreakReason.YieldedAutomatically) { + // acknowledge the yield so the machine can carry on + machine.receiveCmioRequest(); + } + // any other reason (a soft yield or console I/O) just keeps going } - return null; - } catch { - return null; + + let exitCode = 0; + if (breakReason === BreakReason.Halted) { + exitCode = Number(machine.readReg(Reg.HtifToHostData) >> 1n); + } else if (breakReason === BreakReason.McycleOverflow) { + exitCode = 1; + } + + const rootHash: Hash | undefined = options.finalHash + ? (bytesToHex(machine.getRootHash()) as Hash) + : undefined; + + if (options.store) { + machine.store(options.store); + } + + if (options.assertRollingTemplate && exitCode === 0) { + // the machine must be sitting at a rollup accept, waiting for input + try { + const { cmd, reason } = machine.receiveCmioRequest(); + if ( + cmd !== HtifYieldCommand.Manual || + reason !== HtifYieldReason.ManualRxAccepted + ) { + exitCode = 2; + } + } catch { + exitCode = 2; + } + } + + const stdout = outputFile + ? fs.readFileSync(outputFile.name, "utf-8") + : undefined; + + return { breakReason, exitCode, rootHash, stdout }; + } finally { + machine.destroy(); + outputFile?.removeCallback(); } }; +/** + * Version of the machine emulator the bindings were linked against. It is + * fixed at build time, so this is a plain lookup and not a subprocess call + * anymore. + */ +export const version = (): SemVer | null => { + // encoded as (major * 1000000) + (minor * 1000) + patch + const encoded = getVersion(); + const major = encoded / 1000000n; + const minor = (encoded / 1000n) % 1000n; + const patch = encoded % 1000n; + return parse(`${major}.${minor}.${patch}`); +}; + export class UnsupportedVersionError extends Error { constructor(found: SemVer) { super( @@ -42,8 +166,8 @@ export class UnsupportedVersionError extends Error { /** * Throws unless `found` satisfies `requiredVersion`. A version that could not be determined is - * deliberately not an error: `version` also returns null when the binary is missing or docker is - * unavailable, and booting reports those on its own. + * deliberately not an error: `version` also returns null when the emulator the bindings linked + * against reports something unparseable, and creating the machine reports that on its own. */ export const assertSupported = (found: SemVer | null): void => { if (found !== null && !satisfies(found.format(), requiredVersion)) { @@ -52,8 +176,6 @@ export const assertSupported = (found: SemVer | null): void => { }; /** - * Throws if the cartesi-machine that would be used does not satisfy `requiredVersion`. + * Throws if the emulator the bindings linked against does not satisfy `requiredVersion`. */ -export const assertVersion = async ( - options?: ExecaOptionsDockerFallback, -): Promise => assertSupported(await version(options)); +export const assertVersion = (): void => assertSupported(version()); diff --git a/apps/cli/src/images.ts b/apps/cli/src/images.ts new file mode 100644 index 00000000..87a251f7 --- /dev/null +++ b/apps/cli/src/images.ts @@ -0,0 +1,156 @@ +import bytes from "bytes"; +import fs from "fs-extra"; +import { createHash } from "node:crypto"; +import os from "node:os"; +import path from "node:path"; +import type { Reporter } from "./exec/util.js"; + +/** + * Version of https://github.com/cartesi/machine-linux-image the machine boots + * by default, matched to the emulator version @cartesi/machine is linked against. + */ +export const DEFAULT_LINUX_IMAGE_VERSION = "0.21.0"; +export const DEFAULT_LINUX_KERNEL_VERSION = "6.5.13-ctsi-2"; +const DEFAULT_LINUX_KERNEL_SHA256 = + "5c900060da2db2bfa84cd39cd9cd722988c83c42225f3cac55f2d3157e48f32f"; + +const RELEASE_URL = `https://github.com/cartesi/machine-linux-image/releases/download/v${DEFAULT_LINUX_IMAGE_VERSION}/linux-${DEFAULT_LINUX_KERNEL_VERSION}-v${DEFAULT_LINUX_IMAGE_VERSION}.bin`; + +export class ImageDownloadError extends Error { + constructor(url: string, reason: string) { + super(`Failed to download ${url}: ${reason}`); + this.name = "ImageDownloadError"; + } +} + +export class ImageChecksumError extends Error { + constructor(filename: string, expected: string, actual: string) { + super( + `Checksum mismatch for ${filename}: expected ${expected}, got ${actual}`, + ); + this.name = "ImageChecksumError"; + } +} + +/** + * Directory the CLI caches downloaded machine images in. Honors XDG_CACHE_HOME + * when set, and falls back to the platform home directory otherwise. + */ +export const getCacheDir = (): string => { + const base = + process.env.XDG_CACHE_HOME ?? path.join(os.homedir(), ".cache"); + return path.join(base, "cartesi", "images"); +}; + +const sha256 = async (filename: string): Promise => { + const hash = createHash("sha256"); + for await (const chunk of fs.createReadStream(filename)) { + hash.update(chunk as Buffer); + } + return hash.digest("hex"); +}; + +const download = async ( + url: string, + destination: string, + checksum: string, + reporter?: Reporter, +): Promise => { + const response = await fetch(url); + if (!response.ok || !response.body) { + throw new ImageDownloadError( + url, + `${response.status} ${response.statusText}`, + ); + } + + const total = Number(response.headers.get("content-length") ?? 0); + await fs.mkdirp(path.dirname(destination)); + + // download to a temporary file next to the destination, so a partial or + // corrupt download is never mistaken for a cached image + const partial = `${destination}.${process.pid}.part`; + try { + const hash = createHash("sha256"); + let downloaded = 0; + let reported = -1; + const file = fs.createWriteStream(partial); + try { + for await (const chunk of response.body) { + const buffer = chunk as Uint8Array; + hash.update(buffer); + downloaded += buffer.byteLength; + if (!file.write(buffer)) { + await new Promise((resolve) => file.once("drain", resolve)); + } + // report every 10%, chunks are far too small to be worth a + // line each + const percent = + total > 0 ? Math.floor((downloaded * 10) / total) * 10 : -1; + if (reporter && percent > reported) { + reported = percent; + reporter( + `Downloading ${path.basename(destination)}: ${bytes(downloaded)} of ${bytes(total)} (${percent}%)`, + ); + } + } + } finally { + await new Promise((resolve, reject) => + file.end((error?: Error) => + error ? reject(error) : resolve(), + ), + ); + } + + const actual = hash.digest("hex"); + if (actual !== checksum) { + throw new ImageChecksumError( + path.basename(destination), + checksum, + actual, + ); + } + + await fs.move(partial, destination, { overwrite: true }); + } finally { + await fs.remove(partial); + } +}; + +/** + * Resolves the Linux kernel image the machine boots from, downloading it into + * the cache directory on first use. + * + * The lookup order is the CARTESI_IMAGES_PATH directory (the same environment + * variable the cartesi-machine CLI honors), then the cache directory, and + * finally the pinned machine-linux-image release. + */ +export const getRamImage = async (reporter?: Reporter): Promise => { + const filename = `linux-${DEFAULT_LINUX_KERNEL_VERSION}-v${DEFAULT_LINUX_IMAGE_VERSION}.bin`; + + const imagesPath = process.env.CARTESI_IMAGES_PATH; + if (imagesPath) { + for (const candidate of [ + path.join(imagesPath, filename), + path.join(imagesPath, "linux.bin"), + ]) { + if (await fs.pathExists(candidate)) { + return candidate; + } + } + } + + const cached = path.join(getCacheDir(), filename); + if (await fs.pathExists(cached)) { + // a cached image with the wrong contents is a corrupt cache, not a + // reason to fail: drop it and download again + if ((await sha256(cached)) === DEFAULT_LINUX_KERNEL_SHA256) { + return cached; + } + await fs.remove(cached); + } + + reporter?.(`Downloading Linux kernel image ${filename}`); + await download(RELEASE_URL, cached, DEFAULT_LINUX_KERNEL_SHA256, reporter); + return cached; +}; diff --git a/apps/cli/src/machine.ts b/apps/cli/src/machine.ts index 853e208e..58a9cfec 100644 --- a/apps/cli/src/machine.ts +++ b/apps/cli/src/machine.ts @@ -1,82 +1,224 @@ +import type { + MachineConfig as EmulatorMachineConfig, + MachineRuntimeConfig, + MemoryRangeConfig, +} from "@cartesi/machine"; import dotenv from "dotenv"; import fs from "node:fs"; -import { - type Config, - type DriveConfig, - type ImageInfo, - type NvramConfig, - nvramHasImage, - nvramImageFilename, -} from "./config.js"; +import path from "node:path"; +import type { Config, DriveConfig, ImageInfo, NvramConfig } from "./config.js"; +import { nvramHasImage, nvramImageFilename } from "./config.js"; import { cartesiMachine } from "./exec/index.js"; -import type { ExecaOptionsDockerFallback } from "./exec/util.js"; +import type { Reporter } from "./exec/util.js"; +import { getRamImage } from "./images.js"; -const flashDrive = (label: string, drive: DriveConfig): string => { - const { format, mount, shared, user } = drive; - const filename = `${label}.${format}`; - const vars = [`label:${label}`, `data_filename:${filename}`]; - if (mount !== undefined) { - vars.push(`mount:${mount}`); +export class InvalidMemorySizeError extends Error { + constructor(value: string) { + super(`Invalid memory size: ${value}`); + this.name = "InvalidMemorySizeError"; } - if (user) { - vars.push(`user:${user}`); - } - if (shared) { - vars.push("shared"); +} + +export class InvalidEnvNameError extends Error { + constructor(name: string) { + super(`Invalid environment variable name: ${name}`); + this.name = "InvalidEnvNameError"; } - // don't specify start and length - return `--flash-drive=${vars.join(",")}`; +} + +const SHIFTS: Record = { + Ki: 10n, + Mi: 20n, + Gi: 30n, + Ti: 40n, }; -const nvram = (label: string, config: NvramConfig): string => { - const { shared, size, user } = config; - const vars = [`label:${label}`]; - if (size !== undefined) { - vars.push(`length:${size}`); +/** + * Parses a memory size the way the cartesi-machine CLI does: a decimal or + * 0x-prefixed hexadecimal integer, optionally followed by a Ki/Mi/Gi/Ti + * suffix or a `<< n` shift. + */ +export const parseMemorySize = (value: string): bigint => { + const match = value.trim().match(/^(0[xX][0-9a-fA-F]+|[0-9]+)\s*(.*)$/); + if (!match) { + throw new InvalidMemorySizeError(value); } - if (nvramHasImage(config)) { - vars.push(`data_filename:${nvramImageFilename(label)}`); + const [, literal, suffix] = match; + const size = BigInt( + literal.toLowerCase().startsWith("0x") + ? literal.toLowerCase() + : literal.replace(/^0+(?=[0-9])/, ""), + ); + + const rest = suffix.trim(); + let shift = 0n; + if (rest !== "") { + const shiftMatch = rest.match(/^<<\s*([0-9]+)$/); + if (rest in SHIFTS) { + shift = SHIFTS[rest]; + } else if (shiftMatch) { + shift = BigInt(shiftMatch[1]); + } else { + throw new InvalidMemorySizeError(value); + } } - if (user) { - vars.push(`user:${user}`); + + if (size === 0n) { + return 0n; } - if (shared) { - vars.push("shared"); + if (shift >= 64n || size >> (64n - shift) !== 0n) { + throw new InvalidMemorySizeError(value); } - // don't specify start, let cartesi-machine place it - return `--nvram=${vars.join(",")}`; + return size << shift; }; -export type BootMachineOptions = { - finalHash?: boolean; - interactive?: boolean; - store?: string; +/** + * Splash the cartesi-machine CLI prints from init on boot. Every backslash is + * doubled so `echo` emits the art verbatim. + */ +const SPLASH = String.raw`echo " + . + / \\ + / \\ +\\---/---\\ /----\\ + \\ X \\ + \\----/ \\---/---\\ + \\ / CARTESI + \\ / MACHINE + ' +" +`; + +/** + * Mount point of a drive, following the cartesi-machine defaults: a drive + * backed by a file is mounted at /mnt/