diff --git a/.changeset/olive-donkeys-shave.md b/.changeset/olive-donkeys-shave.md new file mode 100644 index 00000000..fe6a5925 --- /dev/null +++ b/.changeset/olive-donkeys-shave.md @@ -0,0 +1,29 @@ +--- +"@cartesi/cli": minor +--- + +Replace the `cartesi-machine` subprocess with the `@cartesi/machine` bindings + +The Cartesi machine is now configured, booted, stored and hashed through +[`@cartesi/machine`](https://github.com/cartesi/rollups-ts), an N-API addon, so `build`, `shell` +and `status` no longer shell out to `cartesi-machine` or `cartesi-machine-stored-hash`, and no +longer fall back to running them inside the SDK Docker image. + +Notable consequences: + +- **The machine emulator moved from 0.20 to 0.21**, which is the version `@cartesi/machine` + links against. Machine hashes change, and applications have to be redeployed. +- **The Linux kernel image is downloaded and cached.** With no SDK image to take it from, the + default `ram_image` now comes from the pinned `cartesi/machine-linux-image` v0.21.0 release, + fetched on first use into `$XDG_CACHE_HOME/cartesi/images` (`~/.cache/cartesi/images`) and + verified against its SHA-256. A `CARTESI_IMAGES_PATH` directory containing the image is used + when set, and `machine.ram_image` in `cartesi.toml` still takes precedence over both. +- **A snapshot is read by the linked emulator.** `cartesi hash` and `cartesi status` no longer + run `cartesi-machine-stored-hash` in the project's SDK image, so a snapshot stored by an + incompatible emulator reads as no hash and has to be rebuilt. +- **Boot args are no longer double quoted.** The old code passed `--append-bootargs=""` to + the CLI without a shell, so the quotes ended up in the kernel command line. They are gone now. +- **The standalone binaries are no longer built or released.** A Bun single file executable has + no `node_modules`, and the addon resolves its platform specific `.node` at runtime, so it + cannot be embedded — not even for the host platform. npm is the only distribution now, and the + homebrew formula has to install the package from there. diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8a3ea37e..efc6deec 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -70,17 +70,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Release CLI binaries - if: ${{ steps.changeset.outputs.published == 'true' && contains(fromJSON(steps.changeset.outputs.publishedPackages).*.name, '@cartesi/cli') }} - run: | - for f in cartesi-*; do tar -czf "$f.tar.gz" "$f"; done - VERSION=$(jq -r '.[] | select(.name=="@cartesi/cli") | .version' <<< '${{ steps.changeset.outputs.publishedPackages }}') - TAG="@cartesi/cli@${VERSION}" - gh release upload "$TAG" cartesi-*.tar.gz - working-directory: ./apps/cli/bin - env: - GH_TOKEN: ${{ github.token }} - build_sdk: name: Build SDK needs: release diff --git a/CLAUDE.md b/CLAUDE.md index 986d1553..520e6c4f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,7 +40,7 @@ bun test apps/cli/tests/unit/config.test.ts # Run a single test bun run build --filter @cartesi/devnet ``` -The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). It also produces native binaries for darwin-arm64, darwin-x64, linux-arm64, linux-x64 in `apps/cli/bin/`. +The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation) → `compile` (Bun bundler → `dist/`). `@cartesi/machine` is left external — it is a native addon that resolves its platform binary at runtime and cannot be bundled, which is also why there are no standalone `bun --compile` binaries. ## Architecture @@ -55,7 +55,9 @@ The CLI build pipeline (`apps/cli`): `clean` → `codegen` (wagmi ABI generation - **`commands/`** — Each file exports a `create*Command()` function returning a Commander command. Main commands: `build`, `run`, `deploy`, `send`, `deposit`, `create`, `doctor`, `shell`, `clean`, `hash`, `logs`, `status`, `address-book`. - **`builder/`** — Drive builder implementations (directory, docker, tar, empty, none). Each builder produces ext2 or SquashFS filesystems for Cartesi Machine drives. - **`compose/`** — Docker Compose service definitions generated as TypeScript objects (anvil, node, bundler, database, paymaster, proxy, explorer, etc.). -- **`exec/`** — Wrappers around subprocess execution (cartesi-machine, rollups) using `execa`. +- **`exec/`** — Machine and filesystem tooling. `cartesi-machine` and `cartesi-machine-stored-hash` are native N-API bindings (`@cartesi/machine`); `genext2fs`, `mksquashfs` and `rollups` still spawn subprocesses via `execa`, falling back to `docker run` against the SDK image. +- **`machine.ts`** — Translates a `cartesi.toml` `Config` into an emulator `MachineConfig` (bootargs, `dtb.init`, flash drives, nvrams), mirroring what the `cartesi-machine` CLI does with its command line. +- **`images.ts`** — Downloads and caches the Linux kernel image the machine boots, from a pinned `cartesi/machine-linux-image` release. - **`config.ts`** — Parses `cartesi.toml` (TOML-based project config) into typed `Config` objects. Defines drive configs, machine configs, and SDK versions. - **`contracts.ts`** — Generated contract addresses and ABI bindings (via `@wagmi/cli`). - **`wallet.ts`** — Wallet utilities using `viem` for Ethereum interaction. diff --git a/apps/cli/build.ts b/apps/cli/build.ts index 3ed24ffb..2bd8b21c 100644 --- a/apps/cli/build.ts +++ b/apps/cli/build.ts @@ -1,35 +1,22 @@ +// the emulator binding resolves its platform binary at runtime, so it can never +// be bundled: it is left as an import, resolved from node_modules +const external = ["@cartesi/machine"]; + // build for npm package await Bun.build({ banner: "#!/usr/bin/env node", entrypoints: ["./src/index.ts"], + external, minify: true, outdir: "dist", sourcemap: true, target: "node", }); -// build bun binaries for all supported platforms -const targets: Bun.Build.CompileTarget[] = [ - "bun-darwin-arm64", - "bun-darwin-x64", - "bun-linux-arm64", - "bun-linux-x64", -]; - -await Promise.all( - targets.map((target) => - Bun.build({ - bytecode: true, - compile: { - outfile: `bin/cartesi-${target.replace("bun-", "")}`, - target, - }, - entrypoints: ["./src/index.ts"], - minify: true, - sourcemap: "linked", - target: "bun", - }), - ), -); +// NOTE: the standalone binaries this used to cross-compile (bin/cartesi-*) +// are gone. A single file executable has no node_modules, and the emulator +// binding resolves its platform specific .node at runtime, so it cannot be +// embedded — not even for the host platform. The npm package is the only +// distribution now, and the homebrew formula has to install it from there. export {}; diff --git a/apps/cli/package.json b/apps/cli/package.json index 8ef34934..e9befd08 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -15,6 +15,7 @@ "/dist" ], "dependencies": { + "@cartesi/machine": "^1.0.0-alpha.2", "@commander-js/extra-typings": "^15.0.0", "@inquirer/confirm": "^6.3.3", "@inquirer/core": "^12.0.4", diff --git a/apps/cli/src/base.ts b/apps/cli/src/base.ts index 4d805326..199e7bd7 100644 --- a/apps/cli/src/base.ts +++ b/apps/cli/src/base.ts @@ -41,33 +41,14 @@ export const getContextPath = (...paths: string[]): string => { }; /** - * SDK image of the project, which built its machine snapshot + * Read the hash of the cartesi machine snapshot, if one exists. The snapshot is + * loaded by the emulator the bindings link against, so one stored by an + * incompatible emulator reads as undefined and has to be rebuilt. */ -const getProjectSdk = (): string | undefined => { - try { - return getApplicationConfig(["cartesi.toml"]).sdk; - } catch { - // an invalid config is reported by the commands that build with it - return undefined; - } -}; - -/** - * Read the hash of the cartesi machine snapshot, if one exists. Without a - * local cartesi-machine-stored-hash, it runs in the SDK image the snapshot was - * built with, as an emulator of another version may not load it. - * @param options sdk image of the project, read from cartesi.toml if not given - */ -export const getMachineHash = async (options?: { - sdk?: string; -}): Promise => { +export const getMachineHash = async (): Promise => { const imagePath = getContextPath("image"); if (fs.existsSync(imagePath)) { - const image = options?.sdk ?? getProjectSdk(); - return await cartesiMachineStoredHash.computeHash( - imagePath, - image ? { image } : undefined, - ); + return await cartesiMachineStoredHash.computeHash(imagePath); } return undefined; }; diff --git a/apps/cli/src/commands/build.ts b/apps/cli/src/commands/build.ts index e805da9d..edb12302 100755 --- a/apps/cli/src/commands/build.ts +++ b/apps/cli/src/commands/build.ts @@ -188,20 +188,30 @@ export const createBuildCommand = () => { } // create machine snapshot - await bootMachine( + const { exitCode, rootHash } = await bootMachine( config, result.imageInfo, { + cwd: destination, finalHash: true, + reporter: (line) => console.error(line), store: "image", }, - { - cwd: destination, - stdio: "inherit", - }, ); - // make snapshot readable by all users, because cartesi-machine sets to 600 + if (exitCode !== 0) { + throw new Error( + exitCode === 2 + ? "Machine did not stop at a rollup accept, it is not a valid rolling template" + : `Machine stopped with exit code ${exitCode}`, + ); + } + + if (rootHash) { + console.error(`Machine hash: ${chalk.cyan(rootHash)}`); + } + + // make snapshot readable by all users, because the emulator sets to 600 await fs.chmod(path.join(destination, "image"), 0o755); }); }; diff --git a/apps/cli/src/commands/doctor.ts b/apps/cli/src/commands/doctor.ts index c54c1d72..920cde63 100755 --- a/apps/cli/src/commands/doctor.ts +++ b/apps/cli/src/commands/doctor.ts @@ -3,7 +3,6 @@ import chalk from "chalk"; import { execa } from "execa"; import ora, { type Ora } from "ora"; import semver from "semver"; -import { DEFAULT_SDK_IMAGE, DEFAULT_SDK_VERSION } from "../config.js"; import { cartesiMachine } from "../exec/index.js"; const MINIMUM_DOCKER_VERSION = "25.0.0"; // Replace with our minimum required Docker version @@ -124,16 +123,12 @@ const checkBuildx = async (progress: Ora): Promise => { const checkCartesiMachine = async (progress: Ora): Promise => { progress.start("Checking Cartesi Machine version..."); - // doctor does not read cartesi.toml, so check against the default sdk image. the host binary - // still takes precedence, which is the install most likely to be out of date - const v = await cartesiMachine.version({ - image: `${DEFAULT_SDK_IMAGE}:${DEFAULT_SDK_VERSION}`, - }); + // the bindings link against the emulator, so this reports the version + // compiled into the CLI rather than probing an install + const v = cartesiMachine.version(); if (v === null) { - throw new Error( - "Could not determine the Cartesi Machine version. Check that Docker is running.", - ); + throw new Error("Could not determine the Cartesi Machine version."); } if (!semver.satisfies(v.format(), cartesiMachine.requiredVersion)) { throw new Error( diff --git a/apps/cli/src/commands/run.ts b/apps/cli/src/commands/run.ts index bc84b331..4f29d554 100755 --- a/apps/cli/src/commands/run.ts +++ b/apps/cli/src/commands/run.ts @@ -83,7 +83,6 @@ const shell = async (options: { projectName: string; prt?: boolean; salt: number; - sdk: string; withdrawalConfig?: WithdrawalConfig; claimStagingPeriod: number; }) => { @@ -93,7 +92,6 @@ const shell = async (options: { log, projectName, prt, - sdk, withdrawalConfig, claimStagingPeriod, } = options; @@ -176,7 +174,7 @@ const shell = async (options: { await build?.parseAsync([], { from: "user" }); // redeploy - const hash = await getMachineHash({ sdk }); + const hash = await getMachineHash(); if (hash) { if (lastDeployment) { await undeploy({ projectName }); @@ -495,9 +493,7 @@ export const createRunCommand = () => { // deploy the application let deployment: RollupsDeployment | undefined; let salt = 0; - const hash = await getMachineHash({ - sdk: applicationConfig.sdk, - }); + const hash = await getMachineHash(); if (hash) { deployment = await deploy({ epochLength, @@ -548,7 +544,6 @@ export const createRunCommand = () => { projectName, prt, salt, - sdk: applicationConfig.sdk, claimStagingPeriod, withdrawalConfig: applicationConfig?.withdrawalConfig, }); diff --git a/apps/cli/src/commands/shell.ts b/apps/cli/src/commands/shell.ts index fb071a86..ed9280d9 100755 --- a/apps/cli/src/commands/shell.ts +++ b/apps/cli/src/commands/shell.ts @@ -1,5 +1,4 @@ import { Command } from "@commander-js/extra-typings"; -import { ExecaError } from "execa"; import fs from "fs-extra"; import path from "node:path"; import { getApplicationConfig, getContextPath } from "../base.js"; @@ -51,26 +50,16 @@ export const createShellCommand = () => { // run as root if flag is set config.machine.user = runAsRoot ? "root" : undefined; - // boot machine - try { - await bootMachine( - config, - undefined, - { interactive: true }, // start with interactive mode on - { - cwd: destination, - stdio: "inherit", - tty: true, - }, - ); - } catch (error: unknown) { - if (error instanceof ExecaError) { - // just continue gracefully - if (error.exitCode === 130) { - return; - } - throw error; - } + // boot machine, in interactive mode + const { exitCode } = await bootMachine(config, undefined, { + cwd: destination, + interactive: true, + reporter: (line) => console.error(line), + }); + + // 130 is the shell being interrupted, which is not a failure + if (exitCode !== 0 && exitCode !== 130) { + throw new Error(`Machine stopped with exit code ${exitCode}`); } }); }; diff --git a/apps/cli/src/config.ts b/apps/cli/src/config.ts index 82ce77e0..12a9b096 100644 --- a/apps/cli/src/config.ts +++ b/apps/cli/src/config.ts @@ -219,9 +219,9 @@ export type MachineConfig = { entrypoint?: string; env: Record; // explicit environment variables injected into cartesi-machine ENV envFile?: string; // path to a .env file with environment variables injected into cartesi-machine ENV - maxMCycle?: bigint; // default given by cartesi-machine + maxMCycle?: bigint; // default is no limit ramLength: string; - ramImage?: string; // default given by cartesi-machine + ramImage?: string; // default is the pinned cartesi machine-linux-image release useDockerEnv: boolean; // inject docker image ENV into cartesi-machine ENV useDockerWorkdir: boolean; // inject docker image WORKDIR into cartesi-machine WORKDIR user?: string; // default given by cartesi-machine diff --git a/apps/cli/src/exec/cartesi-machine-stored-hash.ts b/apps/cli/src/exec/cartesi-machine-stored-hash.ts index 004affd9..85e15eb3 100644 --- a/apps/cli/src/exec/cartesi-machine-stored-hash.ts +++ b/apps/cli/src/exec/cartesi-machine-stored-hash.ts @@ -1,45 +1,21 @@ -import { isHash, type Hash } from "viem"; -import { DEFAULT_SDK_IMAGE, DEFAULT_SDK_VERSION } from "../config.js"; -import { execaDockerFallback, type DockerFallbackOptions } from "./util.js"; - -type ComputeHashOptions = { cwd?: string } & DockerFallbackOptions; +import { load } from "@cartesi/machine"; +import { bytesToHex, type Hash } from "viem"; /** - * - * @param machineDir - * @param options - * @returns + * Reads the root hash of a stored Cartesi machine snapshot. + * @param machineDir directory holding the machine snapshot + * @returns the machine hash, or undefined if the snapshot can't be read */ export const computeHash = async ( machineDir: string, - options?: ComputeHashOptions, ): Promise => { - const defaultImage = `${DEFAULT_SDK_IMAGE}:${DEFAULT_SDK_VERSION}`; - const execaOptions = Object.assign( - {}, - { image: defaultImage, cwd: process.cwd() }, - options, - ); - try { - const { stdout } = await execaDockerFallback( - "cartesi-machine-stored-hash", - [machineDir], - execaOptions, - ); - - if (undefined !== stdout) { - // cartesi-machine-stored-hash prints a bare digest up to emulator - // 0.20 and a 0x-prefixed one from 0.21 on. - const digest = stdout.toString().trim(); - const hash = digest.startsWith("0x") ? digest : `0x${digest}`; - - if (isHash(hash)) { - return hash; - } + const machine = load(machineDir); + try { + return bytesToHex(machine.getRootHash()) as Hash; + } finally { + machine.destroy(); } - - return undefined; } catch { return undefined; } diff --git a/apps/cli/src/exec/cartesi-machine.ts b/apps/cli/src/exec/cartesi-machine.ts index 30571500..5bcb6a93 100644 --- a/apps/cli/src/exec/cartesi-machine.ts +++ b/apps/cli/src/exec/cartesi-machine.ts @@ -1,36 +1,160 @@ -import { parse, Range, satisfies, type SemVer } from "semver"; import { - execaDockerFallback, - type ExecaOptionsDockerFallback, -} from "./util.js"; + BreakReason, + create, + getDefaultConfig, + getVersion, + HtifYieldCommand, + HtifYieldReason, + type MachineConfig, + type MachineRuntimeConfig, + MAX_MCYCLE, + Reg, +} from "@cartesi/machine"; +import { parse, Range, satisfies, type SemVer } from "semver"; +import fs from "node:fs"; +import tmp from "tmp"; +import { bytesToHex, type Hash } from "viem"; export const requiredVersion = new Range("^0.21.0"); -export const boot = ( - args: readonly string[], - options: ExecaOptionsDockerFallback, -) => execaDockerFallback("cartesi-machine", args, options); +export type RunOptions = { + /** fail unless the machine stopped at a rollup accept yield */ + assertRollingTemplate?: boolean; + /** + * collect what the guest writes to its console into `stdout`, instead of + * letting it through to the terminal + */ + captureOutput?: boolean; + /** compute the machine root hash once the run is over */ + finalHash?: boolean; + /** target mcycle to stop at, defaults to no limit */ + maxMCycle?: bigint; + /** directory to store the machine snapshot into */ + store?: string; +}; + +export type RunResult = { + breakReason: BreakReason; + /** exit code of the guest, mirroring what the cartesi-machine CLI reports */ + exitCode: number; + rootHash?: Hash; + /** console output of the guest, when `captureOutput` asked for it */ + stdout?: string; +}; + +/** Machine configuration defaults, as filled in by the emulator itself. */ +export const defaultConfig = (): MachineConfig => getDefaultConfig(); + +/** + * A machine stopped at a halt, a manual yield, or an mcycle overflow no longer + * advances on its own. + */ +const isAtFixedPoint = (breakReason: BreakReason): boolean => + breakReason === BreakReason.Halted || + breakReason === BreakReason.YieldedManually || + breakReason === BreakReason.McycleOverflow; -export const version = async ( - options?: ExecaOptionsDockerFallback, -): Promise => { +/** + * Creates a machine, runs it to a fixed point (or to the requested mcycle), + * and optionally hashes and stores it. Automatic yields are acknowledged and + * discarded, and console I/O breaks just resume the run, which is what the + * cartesi-machine CLI does for a plain boot. + */ +export const run = ( + config: MachineConfig, + runtimeConfig: MachineRuntimeConfig | undefined, + options: RunOptions = {}, +): RunResult => { + // the emulator has no API to read a captured console back, so it writes to + // a file that is read once the run is over. flushing every character keeps + // the file complete without depending on when the machine is torn down + const outputFile = options.captureOutput ? tmp.fileSync() : undefined; + if (outputFile) { + runtimeConfig = { + ...runtimeConfig, + console: { + ...runtimeConfig?.console, + output_destination: "to_file", + output_filename: outputFile.name, + output_flush_mode: "every_char", + }, + }; + } + + const machine = create(config, runtimeConfig); try { - const { stdout } = await execaDockerFallback( - "cartesi-machine", - ["--version-json"], - // cwd is interpolated into the docker volume mount, so it must be defined - { ...options, cwd: options?.cwd ?? process.cwd() }, - ); - if (typeof stdout === "string") { - const output = JSON.parse(stdout); - return parse(output.version); + const target = options.maxMCycle ?? MAX_MCYCLE; + let breakReason: BreakReason; + for (;;) { + breakReason = machine.run(target); + if ( + isAtFixedPoint(breakReason) || + breakReason === BreakReason.ReachedTargetMcycle + ) { + break; + } + if (breakReason === BreakReason.YieldedAutomatically) { + // acknowledge the yield so the machine can carry on + machine.receiveCmioRequest(); + } + // any other reason (a soft yield or console I/O) just keeps going } - return null; - } catch { - return null; + + let exitCode = 0; + if (breakReason === BreakReason.Halted) { + exitCode = Number(machine.readReg(Reg.HtifToHostData) >> 1n); + } else if (breakReason === BreakReason.McycleOverflow) { + exitCode = 1; + } + + const rootHash: Hash | undefined = options.finalHash + ? (bytesToHex(machine.getRootHash()) as Hash) + : undefined; + + if (options.store) { + machine.store(options.store); + } + + if (options.assertRollingTemplate && exitCode === 0) { + // the machine must be sitting at a rollup accept, waiting for input + try { + const { cmd, reason } = machine.receiveCmioRequest(); + if ( + cmd !== HtifYieldCommand.Manual || + reason !== HtifYieldReason.ManualRxAccepted + ) { + exitCode = 2; + } + } catch { + exitCode = 2; + } + } + + const stdout = outputFile + ? fs.readFileSync(outputFile.name, "utf-8") + : undefined; + + return { breakReason, exitCode, rootHash, stdout }; + } finally { + machine.destroy(); + outputFile?.removeCallback(); } }; +/** + * Version of the machine emulator the bindings were linked against. It is + * fixed at build time, so this is a plain lookup and not a subprocess call + * anymore. + */ +export const version = (): SemVer | null => { + // encoded as (major * 1000000) + (minor * 1000) + patch + const encoded = getVersion(); + const major = encoded / 1000000n; + const minor = (encoded / 1000n) % 1000n; + const patch = encoded % 1000n; + return parse(`${major}.${minor}.${patch}`); +}; + export class UnsupportedVersionError extends Error { constructor(found: SemVer) { super( @@ -42,8 +166,8 @@ export class UnsupportedVersionError extends Error { /** * Throws unless `found` satisfies `requiredVersion`. A version that could not be determined is - * deliberately not an error: `version` also returns null when the binary is missing or docker is - * unavailable, and booting reports those on its own. + * deliberately not an error: `version` also returns null when the emulator the bindings linked + * against reports something unparseable, and creating the machine reports that on its own. */ export const assertSupported = (found: SemVer | null): void => { if (found !== null && !satisfies(found.format(), requiredVersion)) { @@ -52,8 +176,6 @@ export const assertSupported = (found: SemVer | null): void => { }; /** - * Throws if the cartesi-machine that would be used does not satisfy `requiredVersion`. + * Throws if the emulator the bindings linked against does not satisfy `requiredVersion`. */ -export const assertVersion = async ( - options?: ExecaOptionsDockerFallback, -): Promise => assertSupported(await version(options)); +export const assertVersion = (): void => assertSupported(version()); diff --git a/apps/cli/src/images.ts b/apps/cli/src/images.ts new file mode 100644 index 00000000..87a251f7 --- /dev/null +++ b/apps/cli/src/images.ts @@ -0,0 +1,156 @@ +import bytes from "bytes"; +import fs from "fs-extra"; +import { createHash } from "node:crypto"; +import os from "node:os"; +import path from "node:path"; +import type { Reporter } from "./exec/util.js"; + +/** + * Version of https://github.com/cartesi/machine-linux-image the machine boots + * by default, matched to the emulator version @cartesi/machine is linked against. + */ +export const DEFAULT_LINUX_IMAGE_VERSION = "0.21.0"; +export const DEFAULT_LINUX_KERNEL_VERSION = "6.5.13-ctsi-2"; +const DEFAULT_LINUX_KERNEL_SHA256 = + "5c900060da2db2bfa84cd39cd9cd722988c83c42225f3cac55f2d3157e48f32f"; + +const RELEASE_URL = `https://github.com/cartesi/machine-linux-image/releases/download/v${DEFAULT_LINUX_IMAGE_VERSION}/linux-${DEFAULT_LINUX_KERNEL_VERSION}-v${DEFAULT_LINUX_IMAGE_VERSION}.bin`; + +export class ImageDownloadError extends Error { + constructor(url: string, reason: string) { + super(`Failed to download ${url}: ${reason}`); + this.name = "ImageDownloadError"; + } +} + +export class ImageChecksumError extends Error { + constructor(filename: string, expected: string, actual: string) { + super( + `Checksum mismatch for ${filename}: expected ${expected}, got ${actual}`, + ); + this.name = "ImageChecksumError"; + } +} + +/** + * Directory the CLI caches downloaded machine images in. Honors XDG_CACHE_HOME + * when set, and falls back to the platform home directory otherwise. + */ +export const getCacheDir = (): string => { + const base = + process.env.XDG_CACHE_HOME ?? path.join(os.homedir(), ".cache"); + return path.join(base, "cartesi", "images"); +}; + +const sha256 = async (filename: string): Promise => { + const hash = createHash("sha256"); + for await (const chunk of fs.createReadStream(filename)) { + hash.update(chunk as Buffer); + } + return hash.digest("hex"); +}; + +const download = async ( + url: string, + destination: string, + checksum: string, + reporter?: Reporter, +): Promise => { + const response = await fetch(url); + if (!response.ok || !response.body) { + throw new ImageDownloadError( + url, + `${response.status} ${response.statusText}`, + ); + } + + const total = Number(response.headers.get("content-length") ?? 0); + await fs.mkdirp(path.dirname(destination)); + + // download to a temporary file next to the destination, so a partial or + // corrupt download is never mistaken for a cached image + const partial = `${destination}.${process.pid}.part`; + try { + const hash = createHash("sha256"); + let downloaded = 0; + let reported = -1; + const file = fs.createWriteStream(partial); + try { + for await (const chunk of response.body) { + const buffer = chunk as Uint8Array; + hash.update(buffer); + downloaded += buffer.byteLength; + if (!file.write(buffer)) { + await new Promise((resolve) => file.once("drain", resolve)); + } + // report every 10%, chunks are far too small to be worth a + // line each + const percent = + total > 0 ? Math.floor((downloaded * 10) / total) * 10 : -1; + if (reporter && percent > reported) { + reported = percent; + reporter( + `Downloading ${path.basename(destination)}: ${bytes(downloaded)} of ${bytes(total)} (${percent}%)`, + ); + } + } + } finally { + await new Promise((resolve, reject) => + file.end((error?: Error) => + error ? reject(error) : resolve(), + ), + ); + } + + const actual = hash.digest("hex"); + if (actual !== checksum) { + throw new ImageChecksumError( + path.basename(destination), + checksum, + actual, + ); + } + + await fs.move(partial, destination, { overwrite: true }); + } finally { + await fs.remove(partial); + } +}; + +/** + * Resolves the Linux kernel image the machine boots from, downloading it into + * the cache directory on first use. + * + * The lookup order is the CARTESI_IMAGES_PATH directory (the same environment + * variable the cartesi-machine CLI honors), then the cache directory, and + * finally the pinned machine-linux-image release. + */ +export const getRamImage = async (reporter?: Reporter): Promise => { + const filename = `linux-${DEFAULT_LINUX_KERNEL_VERSION}-v${DEFAULT_LINUX_IMAGE_VERSION}.bin`; + + const imagesPath = process.env.CARTESI_IMAGES_PATH; + if (imagesPath) { + for (const candidate of [ + path.join(imagesPath, filename), + path.join(imagesPath, "linux.bin"), + ]) { + if (await fs.pathExists(candidate)) { + return candidate; + } + } + } + + const cached = path.join(getCacheDir(), filename); + if (await fs.pathExists(cached)) { + // a cached image with the wrong contents is a corrupt cache, not a + // reason to fail: drop it and download again + if ((await sha256(cached)) === DEFAULT_LINUX_KERNEL_SHA256) { + return cached; + } + await fs.remove(cached); + } + + reporter?.(`Downloading Linux kernel image ${filename}`); + await download(RELEASE_URL, cached, DEFAULT_LINUX_KERNEL_SHA256, reporter); + return cached; +}; diff --git a/apps/cli/src/machine.ts b/apps/cli/src/machine.ts index 853e208e..58a9cfec 100644 --- a/apps/cli/src/machine.ts +++ b/apps/cli/src/machine.ts @@ -1,82 +1,224 @@ +import type { + MachineConfig as EmulatorMachineConfig, + MachineRuntimeConfig, + MemoryRangeConfig, +} from "@cartesi/machine"; import dotenv from "dotenv"; import fs from "node:fs"; -import { - type Config, - type DriveConfig, - type ImageInfo, - type NvramConfig, - nvramHasImage, - nvramImageFilename, -} from "./config.js"; +import path from "node:path"; +import type { Config, DriveConfig, ImageInfo, NvramConfig } from "./config.js"; +import { nvramHasImage, nvramImageFilename } from "./config.js"; import { cartesiMachine } from "./exec/index.js"; -import type { ExecaOptionsDockerFallback } from "./exec/util.js"; +import type { Reporter } from "./exec/util.js"; +import { getRamImage } from "./images.js"; -const flashDrive = (label: string, drive: DriveConfig): string => { - const { format, mount, shared, user } = drive; - const filename = `${label}.${format}`; - const vars = [`label:${label}`, `data_filename:${filename}`]; - if (mount !== undefined) { - vars.push(`mount:${mount}`); +export class InvalidMemorySizeError extends Error { + constructor(value: string) { + super(`Invalid memory size: ${value}`); + this.name = "InvalidMemorySizeError"; } - if (user) { - vars.push(`user:${user}`); - } - if (shared) { - vars.push("shared"); +} + +export class InvalidEnvNameError extends Error { + constructor(name: string) { + super(`Invalid environment variable name: ${name}`); + this.name = "InvalidEnvNameError"; } - // don't specify start and length - return `--flash-drive=${vars.join(",")}`; +} + +const SHIFTS: Record = { + Ki: 10n, + Mi: 20n, + Gi: 30n, + Ti: 40n, }; -const nvram = (label: string, config: NvramConfig): string => { - const { shared, size, user } = config; - const vars = [`label:${label}`]; - if (size !== undefined) { - vars.push(`length:${size}`); +/** + * Parses a memory size the way the cartesi-machine CLI does: a decimal or + * 0x-prefixed hexadecimal integer, optionally followed by a Ki/Mi/Gi/Ti + * suffix or a `<< n` shift. + */ +export const parseMemorySize = (value: string): bigint => { + const match = value.trim().match(/^(0[xX][0-9a-fA-F]+|[0-9]+)\s*(.*)$/); + if (!match) { + throw new InvalidMemorySizeError(value); } - if (nvramHasImage(config)) { - vars.push(`data_filename:${nvramImageFilename(label)}`); + const [, literal, suffix] = match; + const size = BigInt( + literal.toLowerCase().startsWith("0x") + ? literal.toLowerCase() + : literal.replace(/^0+(?=[0-9])/, ""), + ); + + const rest = suffix.trim(); + let shift = 0n; + if (rest !== "") { + const shiftMatch = rest.match(/^<<\s*([0-9]+)$/); + if (rest in SHIFTS) { + shift = SHIFTS[rest]; + } else if (shiftMatch) { + shift = BigInt(shiftMatch[1]); + } else { + throw new InvalidMemorySizeError(value); + } } - if (user) { - vars.push(`user:${user}`); + + if (size === 0n) { + return 0n; } - if (shared) { - vars.push("shared"); + if (shift >= 64n || size >> (64n - shift) !== 0n) { + throw new InvalidMemorySizeError(value); } - // don't specify start, let cartesi-machine place it - return `--nvram=${vars.join(",")}`; + return size << shift; }; -export type BootMachineOptions = { - finalHash?: boolean; - interactive?: boolean; - store?: string; +/** + * Splash the cartesi-machine CLI prints from init on boot. Every backslash is + * doubled so `echo` emits the art verbatim. + */ +const SPLASH = String.raw`echo " + . + / \\ + / \\ +\\---/---\\ /----\\ + \\ X \\ + \\----/ \\---/---\\ + \\ / CARTESI + \\ / MACHINE + ' +" +`; + +/** + * Mount point of a drive, following the cartesi-machine defaults: a drive + * backed by a file is mounted at /mnt/