From 16f070fa88bb82f915ecc13a65d02aa30e669925 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 1 Jul 2026 10:56:43 +0200 Subject: [PATCH 1/6] feat(vitest-plugin): support vitest 5 alongside 3 and 4 Vitest 5 reworked the benchmark backend: the dedicated `NodeBenchmarkRunner` and the `vitest/runners` / `vitest/suite` entrypoints are gone, benchmarks now run inside `test()` through the unified `TestRunner`, and tinybench moved to v6 (stats moved from `result.benchmark` to `task.result.latency`, `includeSamples` became `retainSamples`). Detect the installed Vitest generation and select the integration seam behind a `VitestBackend` abstraction so the rest of the plugin never inspects the version: - v3/4 keep the custom benchmark runner per instrument mode (analysis/walltime). - v5 installs instrumentation from a setup file that patches the shared `TestRunner.runBenchmarks` static. A setup file (rather than a custom `test.runner`) leaves the runner untouched for non-benchmark tests and also applies to the browser pool. --- packages/vitest-plugin/package.json | 4 +- packages/vitest-plugin/rollup.config.mjs | 16 +- .../vitest-plugin/src/__tests__/index.test.ts | 99 ++++++- .../src/__tests__/instrumented.test.ts | 13 +- packages/vitest-plugin/src/index.ts | 74 ++--- packages/vitest-plugin/src/instrument.ts | 258 ++++++++++++++++++ .../src/{ => legacy}/analysis.ts | 9 +- .../vitest-plugin/src/{ => legacy}/common.ts | 5 +- .../vitest-plugin/src/{ => legacy}/compat.ts | 0 .../src/legacy/vitest-legacy.d.ts | 46 ++++ .../src/legacy/walltime-utils.ts | 110 ++++++++ packages/vitest-plugin/src/legacy/walltime.ts | 119 ++++++++ packages/vitest-plugin/src/runner.ts | 3 - packages/vitest-plugin/src/v5/setup.ts | 131 +++++++++ packages/vitest-plugin/src/vitestBackend.ts | 152 +++++++++++ packages/vitest-plugin/src/walltime/index.ts | 216 --------------- packages/vitest-plugin/src/walltime/utils.ts | 130 --------- packages/vitest-plugin/vitest.config.ts | 14 +- 18 files changed, 970 insertions(+), 429 deletions(-) create mode 100644 packages/vitest-plugin/src/instrument.ts rename packages/vitest-plugin/src/{ => legacy}/analysis.ts (90%) rename packages/vitest-plugin/src/{ => legacy}/common.ts (83%) rename packages/vitest-plugin/src/{ => legacy}/compat.ts (100%) create mode 100644 packages/vitest-plugin/src/legacy/vitest-legacy.d.ts create mode 100644 packages/vitest-plugin/src/legacy/walltime-utils.ts create mode 100644 packages/vitest-plugin/src/legacy/walltime.ts delete mode 100644 packages/vitest-plugin/src/runner.ts create mode 100644 packages/vitest-plugin/src/v5/setup.ts create mode 100644 packages/vitest-plugin/src/vitestBackend.ts delete mode 100644 packages/vitest-plugin/src/walltime/index.ts delete mode 100644 packages/vitest-plugin/src/walltime/utils.ts diff --git a/packages/vitest-plugin/package.json b/packages/vitest-plugin/package.json index 3c935d88..ab23e739 100644 --- a/packages/vitest-plugin/package.json +++ b/packages/vitest-plugin/package.json @@ -41,13 +41,13 @@ "peerDependencies": { "tinybench": ">=2.9.0", "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0", - "vitest": "^3.2 || ^4" + "vitest": "^3.2 || ^4 || ^5" }, "devDependencies": { "@total-typescript/shoehorn": "^0.1.1", "execa": "^8.0.1", "tinybench": "^2.9.0", "vite": "^8.0.0", - "vitest": "^4.1.11" + "vitest": "^5.0.0" } } diff --git a/packages/vitest-plugin/rollup.config.mjs b/packages/vitest-plugin/rollup.config.mjs index 2fda2b6a..6c9827c7 100644 --- a/packages/vitest-plugin/rollup.config.mjs +++ b/packages/vitest-plugin/rollup.config.mjs @@ -23,18 +23,26 @@ export default defineConfig([ plugins: jsPlugins(pkg.version), external: ["@codspeed/core", /^vitest/], }, + // The built layout mirrors the source layout (dist/legacy/*, dist/v5/*) so the + // plugin resolves the seam files with one path rule in both dev and prod. { - input: "src/analysis.ts", - output: { file: "dist/analysis.mjs", format: "es" }, + input: "src/legacy/analysis.ts", + output: { file: "dist/legacy/analysis.mjs", format: "es" }, // top-level await plugins: jsPlugins(pkg.version, "es2022"), external: ["@codspeed/core", /^vitest/], }, { - input: "src/walltime/index.ts", - output: { file: "dist/walltime.mjs", format: "es" }, + input: "src/legacy/walltime.ts", + output: { file: "dist/legacy/walltime.mjs", format: "es" }, // top-level await plugins: jsPlugins(pkg.version, "es2022"), external: ["@codspeed/core", /^vitest/], }, + { + input: "src/v5/setup.ts", + output: { file: "dist/v5/setup.mjs", format: "es" }, + plugins: jsPlugins(pkg.version), + external: ["@codspeed/core", /^vitest/, "tinybench"], + }, ]); diff --git a/packages/vitest-plugin/src/__tests__/index.test.ts b/packages/vitest-plugin/src/__tests__/index.test.ts index 63ce93cc..f129c229 100644 --- a/packages/vitest-plugin/src/__tests__/index.test.ts +++ b/packages/vitest-plugin/src/__tests__/index.test.ts @@ -55,6 +55,7 @@ describe("codSpeedPlugin", () => { // Clean up environment variables delete process.env.CODSPEED_ENV; delete process.env.CODSPEED_RUNNER_MODE; + fsMocks.setMockVersion("4.0.18"); }); it("should have a name", async () => { @@ -66,7 +67,9 @@ describe("codSpeedPlugin", () => { }); describe("apply", () => { - it("should not apply the plugin when the mode is not benchmark", async () => { + it("should not apply the plugin when the mode is not benchmark (v3/v4)", async () => { + fsMocks.setMockVersion("4.0.18"); + const applyPlugin = applyPluginFunction( {}, fromPartial({ mode: "test" }), @@ -75,7 +78,8 @@ describe("codSpeedPlugin", () => { expect(applyPlugin).toBe(false); }); - it("should apply the plugin when there is no instrumentation", async () => { + it("should apply the plugin when there is no instrumentation (v3/v4)", async () => { + fsMocks.setMockVersion("4.0.18"); coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(false); const applyPlugin = applyPluginFunction( @@ -89,7 +93,8 @@ describe("codSpeedPlugin", () => { expect(applyPlugin).toBe(true); }); - it("should apply the plugin when there is instrumentation", async () => { + it("should apply the plugin when there is instrumentation (v3/v4)", async () => { + fsMocks.setMockVersion("4.0.18"); coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true); const applyPlugin = applyPluginFunction( @@ -99,14 +104,32 @@ describe("codSpeedPlugin", () => { expect(applyPlugin).toBe(true); }); + + it("should stay active regardless of mode on v5 (benchmark gating happens in config)", async () => { + fsMocks.setMockVersion("5.0.0-beta.5"); + coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true); + + const applyPlugin = applyPluginFunction( + {}, + fromPartial({ mode: "test" }), + ); + + expect(applyPlugin).toBe(true); + fsMocks.setMockVersion("4.0.18"); + }); }); it("should apply the codspeed config for v4", () => { + fsMocks.setMockVersion("4.0.18"); const config = resolvedCodSpeedPlugin.config; if (typeof config !== "function") throw new Error("config is not a function"); - const result = config.call({} as never, {}, fromPartial({})); + const result = config.call( + {} as never, + {}, + fromPartial({ mode: "benchmark" }), + ); expect(result).toStrictEqual({ test: { @@ -116,23 +139,25 @@ describe("codSpeedPlugin", () => { pool: "forks", execArgv: getV8Flags(), runner: expect.stringContaining( - "packages/vitest-plugin/src/analysis.ts", + "packages/vitest-plugin/src/legacy/analysis.ts", ), }, }); }); it("should apply the codspeed config for v3 with poolOptions", () => { - // Set mock version to v3 fsMocks.setMockVersion("3.2.0"); - // Create a new plugin instance to pick up the mocked version const v3Plugin = codspeedPlugin(); const config = v3Plugin.config; if (typeof config !== "function") throw new Error("config is not a function"); - const result = config.call({} as never, {}, fromPartial({})); + const result = config.call( + {} as never, + {}, + fromPartial({ mode: "benchmark" }), + ); expect(result).toStrictEqual({ test: { @@ -146,12 +171,66 @@ describe("codSpeedPlugin", () => { }, }, runner: expect.stringContaining( - "packages/vitest-plugin/src/analysis.ts", + "packages/vitest-plugin/src/legacy/analysis.ts", ), }, }); - // Reset mock version back to v4 fsMocks.setMockVersion("4.0.18"); }); + + describe("v5 config", () => { + it("should not inject config when benchmarks are not enabled", () => { + fsMocks.setMockVersion("5.0.0-beta.5"); + const v5Plugin = codspeedPlugin(); + const config = v5Plugin.config; + if (typeof config !== "function") + throw new Error("config is not a function"); + + const result = config.call( + {} as never, + {}, + fromPartial({ mode: "test" }), + ); + + expect(result).toBeUndefined(); + fsMocks.setMockVersion("4.0.18"); + }); + + it("should inject the v5 setup file (not a runner) when benchmarks are enabled", () => { + fsMocks.setMockVersion("5.0.0-beta.5"); + const v5Plugin = codspeedPlugin(); + const config = v5Plugin.config; + if (typeof config !== "function") + throw new Error("config is not a function"); + + const result = config.call( + {} as never, + // `benchmark.enabled` is a Vitest 5 config field the v3/4 typings (which + // this file may be compiled against) don't expose. + { test: { benchmark: { enabled: true } } } as never, + fromPartial({ mode: "test" }), + ); + + expect(result).toStrictEqual({ + test: { + globalSetup: [ + expect.stringContaining( + "packages/vitest-plugin/src/globalSetup.ts", + ), + ], + pool: "forks", + execArgv: getV8Flags(), + setupFiles: [ + expect.stringContaining("packages/vitest-plugin/src/v5/setup.ts"), + ], + }, + }); + // The v5 path must not set a custom runner. + expect( + (result as { test?: { runner?: unknown } })?.test?.runner, + ).toBeUndefined(); + fsMocks.setMockVersion("4.0.18"); + }); + }); }); diff --git a/packages/vitest-plugin/src/__tests__/instrumented.test.ts b/packages/vitest-plugin/src/__tests__/instrumented.test.ts index 443d62f2..a3106544 100644 --- a/packages/vitest-plugin/src/__tests__/instrumented.test.ts +++ b/packages/vitest-plugin/src/__tests__/instrumented.test.ts @@ -1,7 +1,12 @@ import { fromPartial } from "@total-typescript/shoehorn"; import { describe, expect, it, vi, type RunnerTestSuite } from "vitest"; -import { AnalysisRunner as CodSpeedRunner } from "../analysis"; -import { getBenchFn, getBenchOptions } from "../compat"; +import { AnalysisRunner as CodSpeedRunner } from "../legacy/analysis"; +import { getBenchFn, getBenchOptions } from "../legacy/compat"; + +// The legacy AnalysisRunner targets the Vitest 3/4 benchmark backend +// (`NodeBenchmarkRunner`, `vitest/suite`), which Vitest 5 removed. This whole +// file is excluded from the test run under v5+ (see vitest.config.ts); the v5 +// path is covered separately. const coreMocks = vi.hoisted(() => { return { @@ -28,8 +33,8 @@ vi.mock("@codspeed/core", async (importOriginal) => { console.log = vi.fn(); -vi.mock("../compat", async (importOriginal) => { - const actual = await importOriginal(); +vi.mock("../legacy/compat", async (importOriginal) => { + const actual = await importOriginal(); return { ...actual, getBenchFn: vi.fn(), diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index 2bdf59ed..c3216cab 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -7,49 +7,33 @@ import { SetupInstrumentsRequestBody, SetupInstrumentsResponse, } from "@codspeed/core"; -import { readFileSync } from "fs"; -import { createRequire } from "module"; import { join } from "path"; import { Plugin } from "vite"; import { type ViteUserConfig } from "vitest/config"; +import { resolveVitestBackend } from "./vitestBackend"; // get this file's directory path from import.meta.url const __dirname = new URL(".", import.meta.url).pathname; const isFileInTs = import.meta.url.endsWith(".ts"); -function getCodSpeedFileFromName(name: string) { +/** + * Resolve a plugin-owned file (globalSetup, seam entry points) shipped alongside + * this module. Source (`.ts`) and built (`.mjs`) layouts are kept identical (see + * rollup.config.ts), so the same relative `name` works in both. + */ +function resolveFile(name: string): string { const fileExtension = isFileInTs ? "ts" : "mjs"; - return join(__dirname, `${name}.${fileExtension}`); } -function getVitestMajorVersion(): number | null { - try { - // Resolve vitest from the project's perspective (cwd), not from the plugin's location - // This ensures we detect the vitest version the user has installed - const require = createRequire(join(process.cwd(), "package.json")); - const vitestPkgPath = require.resolve("vitest/package.json"); - const vitestPkg = JSON.parse(readFileSync(vitestPkgPath, "utf-8")); - return parseInt(vitestPkg.version.split(".")[0], 10); - } catch { - return null; - } -} - -function getRunnerFile(): string | undefined { - const instrumentMode = getInstrumentMode(); - if (instrumentMode === "disabled") { - return undefined; - } - - return getCodSpeedFileFromName(instrumentMode); -} - export default function codspeedPlugin(): Plugin { + // Resolved lazily on each hook rather than once here: the installed Vitest + // version is detected from the project's cwd, which isn't reliably knowable at + // plugin-construction time (and tests swap it between construction and use). return { name: "codspeed:vitest", apply(_, { mode }) { - if (mode !== "benchmark") { + if (!resolveVitestBackend().isActiveForViteMode(mode)) { return false; } if ( @@ -61,37 +45,19 @@ export default function codspeedPlugin(): Plugin { return true; }, enforce: "post", - config(): ViteUserConfig { - const runnerFile = getRunnerFile(); - const runnerMode = getCodspeedRunnerMode(); - const v8Flags = getV8Flags(); - const vitestMajorVersion = getVitestMajorVersion(); - // by default, assume Vitest v4 or higher - const isVitestV4OrHigher = (vitestMajorVersion ?? 4) >= 4; + config(incomingConfig, { mode }): ViteUserConfig | undefined { + const backend = resolveVitestBackend(); + if (!backend.isBenchmarkRun(incomingConfig, mode)) { + return undefined; + } const config: ViteUserConfig = { test: { pool: "forks", - ...(isVitestV4OrHigher - ? { execArgv: v8Flags } - : { - // Compat with Vitest v3 - // See: https://vitest.dev/guide/migration.html#pool-rework - // poolOptions only exists in Vitest v3 - poolOptions: { - forks: { - execArgv: v8Flags, - }, - }, - }), - globalSetup: [getCodSpeedFileFromName("globalSetup")], - ...(runnerFile && { - runner: runnerFile, - }), - ...(runnerMode === "walltime" && { - benchmark: { - includeSamples: true, - }, + globalSetup: [resolveFile("globalSetup")], + ...backend.getBenchmarkTestConfig(getV8Flags(), resolveFile), + ...(getCodspeedRunnerMode() === "walltime" && { + benchmark: backend.getWalltimeBenchmarkConfig(), }), }, }; diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts new file mode 100644 index 00000000..b880ca89 --- /dev/null +++ b/packages/vitest-plugin/src/instrument.ts @@ -0,0 +1,258 @@ +import { + calculateQuantiles, + InstrumentHooks, + MARKER_TYPE_BENCHMARK_END, + MARKER_TYPE_BENCHMARK_START, + msToNs, + msToS, + wrapWithRootFrame, + writeWalltimeResults, + type Benchmark, + type BenchmarkStats, +} from "@codspeed/core"; +import type * as tinybench from "tinybench"; + +export type Tinybench = typeof tinybench; + +/** A tinybench task, exposing the `fn` the runner wraps with the root frame. */ +export interface TinybenchTask { + name: string; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fn: (...args: any[]) => any; + result?: TinybenchTaskResult; +} + +/** tinybench's per-task setup/teardown hook signature. */ +export type TinybenchHook = ( + task: TinybenchTask, + mode: "run" | "warmup", +) => Promise | void; + +/** The mutable subset of a tinybench Bench the runner reaches into. */ +export interface TinybenchBench { + setup: TinybenchHook; + teardown: TinybenchHook; +} + +/** The minimal task shape `patchTaskRunWithRootFrame` mutates. */ +interface RunnableTask { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fn: (...args: any[]) => any; +} + +/** The tinybench Task prototype whose `run` we wrap. */ +interface TinybenchTaskClass { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prototype: { run: (this: any) => Promise }; +} + +/** + * The tinybench statistics shape (latency/throughput) shared across the v2 and + * v6 lines. Only the fields the conversion needs are modeled. + */ +interface TinybenchStatistics { + min: number; + max: number; + mean: number; + sd: number; + samples: number[] | undefined; +} + +interface TinybenchTaskResult { + state?: string; + totalTime: number; + latency: TinybenchStatistics; +} + +/** The subset of tinybench bench options that maps onto a CodSpeed benchmark config. */ +export interface TinybenchOptions { + time?: number; + warmupTime?: number; + warmupIterations?: number; + iterations?: number; +} + +/** Timestamp marking the open edge of a task's measured loop. */ +interface InstrumentWindow { + runStart: bigint | null; +} + +let isTaskPatched = false; + +/** + * The window bracketing the currently running task's measured loop, driven by + * the setup/teardown hooks below. Tasks run strictly sequentially within a + * worker, so one shared value suffices. + */ +const instrumentWindow: InstrumentWindow = { runStart: null }; + +/** + * Wrap every task's fn with the root frame so collected stacks are attributed to + * a benchmark. Idempotent: patching the shared `Task.prototype.run` in place hits + * every Bench instance, so repeat calls are no-ops. + * + * `TaskClass` must be the exact prototype the host constructed its tasks against + * (taken from a live task, not imported) so the patch applies even when multiple + * copies of tinybench are installed. + */ +export function patchTaskRunOnce(TaskClass: TinybenchTaskClass): void { + if (isTaskPatched) { + return; + } + isTaskPatched = true; + + const originalRun = TaskClass.prototype.run; + TaskClass.prototype.run = async function (this: RunnableTask) { + const originalFn = this.fn; + this.fn = wrapWithRootFrame(() => originalFn.call(this)); + + try { + return await originalRun.call(this); + } finally { + this.fn = originalFn; + } + }; +} + +/** + * Drive the instrumentation window from each bench's run-mode setup/teardown + * hooks so it brackets only tinybench's measured loop, excluding the warmup + * that runs beforehand and the statistics computation tinybench performs after + * the loop. Wrapping the whole `Task.run()` would otherwise fold all of that + * framework overhead into the recorded sample. + * + * User-provided hooks are preserved and keep their order relative to the work + * under test. + */ +export function installInstrumentHooks( + bench: TinybenchBench, + getUri: (taskName: string) => string, +): void { + const userSetup = bench.setup; + const userTeardown = bench.teardown; + + bench.setup = async (task, mode) => { + await userSetup(task, mode); + if (mode === "run") { + InstrumentHooks.startBenchmark(); + instrumentWindow.runStart = InstrumentHooks.currentTimestamp(); + } + }; + + bench.teardown = async (task, mode) => { + if (mode === "run") { + closeInstrumentWindow(getUri(task.name)); + } + await userTeardown(task, mode); + }; +} + +function closeInstrumentWindow(uri: string): void { + const runEnd = InstrumentHooks.currentTimestamp(); + const pid = process.pid; + + // Benchmark markers must land inside the sample window opened by + // startBenchmark(), so they have to be emitted before stopBenchmark() + // closes it. The runner consumes the FIFO stream in order, so a marker + // sent after StopBenchmark falls outside the sample and breaks the + // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. + InstrumentHooks.addMarker( + pid, + MARKER_TYPE_BENCHMARK_START, + instrumentWindow.runStart!, + ); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); + + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(pid, uri); + instrumentWindow.runStart = null; +} + +/** + * Persist collected walltime benchmarks, if any. The per-seam result traversal + * differs (legacy walks the Vitest suite tree, v5 iterates the live tinybench + * tasks) because each generation exposes results differently, but the write and + * the summary log are identical. + */ +export function writeAndLogWalltimeResults(benchmarks: Benchmark[]): void { + if (benchmarks.length === 0) { + return; + } + writeWalltimeResults(benchmarks); + console.log( + `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.`, + ); +} + +/** + * Convert a completed tinybench task into a CodSpeed walltime benchmark. Returns + * null when the task produced no samples (e.g. fully optimized out), in which + * case there is nothing to record. + */ +export function tinybenchTaskToBenchmark( + task: TinybenchTask, + uri: string, + options: TinybenchOptions, +): Benchmark | null { + const stats = tinybenchResultToStats(task.result, options); + if (stats === null) { + return null; + } + + return { + name: task.name, + uri, + config: { + max_rounds: options.iterations ?? null, + max_time_ns: options.time ? msToNs(options.time) : null, + min_round_time_ns: null, // tinybench does not have an option for this + warmup_time_ns: + options.warmupIterations !== 0 && options.warmupTime + ? msToNs(options.warmupTime) + : null, + }, + stats, + }; +} + +function tinybenchResultToStats( + result: TinybenchTaskResult | undefined, + options: TinybenchOptions, +): BenchmarkStats | null { + if (!result) { + throw new Error("No benchmark data available in result"); + } + + const { totalTime, latency } = result; + const { min, max, mean, sd, samples } = latency; + + const sortedTimesNs = (samples ?? []).map(msToNs).sort((a, b) => a - b); + const meanNs = msToNs(mean); + const stdevNs = msToNs(sd); + + if (sortedTimesNs.length == 0) { + // Sometimes the benchmarks can be completely optimized out and not even + // run, but their beforeEach and afterEach hooks are still executed, and the + // task is still considered a success. + return null; + } + + const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = + calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); + + return { + min_ns: msToNs(min), + max_ns: msToNs(max), + mean_ns: meanNs, + stdev_ns: stdevNs, + q1_ns, + median_ns, + q3_ns, + total_time: msToS(totalTime), + iter_per_round: 1, // tinybench runs one iteration per round + rounds: sortedTimesNs.length, + iqr_outlier_rounds, + stdev_outlier_rounds, + warmup_iters: options.warmupIterations ?? 0, + }; +} diff --git a/packages/vitest-plugin/src/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts similarity index 90% rename from packages/vitest-plugin/src/analysis.ts rename to packages/vitest-plugin/src/legacy/analysis.ts index dc50a5e2..9821003e 100644 --- a/packages/vitest-plugin/src/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -9,6 +9,7 @@ import { } from "@codspeed/core"; import type * as tinybench from "tinybench"; import { Benchmark, type RunnerTestSuite } from "vitest"; + import { callSuiteHook, isVitestTaskBenchmark, @@ -39,7 +40,11 @@ async function runAnalysisBench( tinybenchModule: Tinybench, ) { const uri = `${currentSuiteName}::${benchmark.name}`; - const fn = getBenchFn(benchmark); + // tinybench's bench fn carries a `this: Bench` requirement on Vitest 3/4 that + // we don't need (the work under test is self-contained); call it as a plain + // parameterless function. The cast also smooths over the typing differences + // across supported Vitest versions. + const fn = getBenchFn(benchmark) as () => unknown; // Constructing a Bench applies tinybench's no-op defaults for the setup and // teardown hooks and gives them the Task they expect. The bench itself is @@ -50,7 +55,6 @@ async function runAnalysisBench( await bench.setup(task, "warmup"); await optimizeFunction(async () => { await callSuiteHook(suite, benchmark, "beforeEach"); - // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench await fn(); await callSuiteHook(suite, benchmark, "afterEach"); }); @@ -62,7 +66,6 @@ async function runAnalysisBench( global.gc?.(); await wrapWithRootFrame(async () => { InstrumentHooks.startBenchmark(); - // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench await fn(); InstrumentHooks.stopBenchmark(); InstrumentHooks.setExecutedBenchmark(process.pid, uri); diff --git a/packages/vitest-plugin/src/common.ts b/packages/vitest-plugin/src/legacy/common.ts similarity index 83% rename from packages/vitest-plugin/src/common.ts rename to packages/vitest-plugin/src/legacy/common.ts index e853c307..a76f9473 100644 --- a/packages/vitest-plugin/src/common.ts +++ b/packages/vitest-plugin/src/legacy/common.ts @@ -19,8 +19,9 @@ export async function callSuiteHook( const hooks = getSuiteHooks(suite, name); - // @ts-expect-error TODO: add support for hooks parameters - await Promise.all(hooks.map((fn) => fn())); + // TODO: add support for hook parameters. The hook signature differs across + // supported Vitest versions, so we call them through a parameterless cast. + await Promise.all((hooks as Array<() => unknown>).map((fn) => fn())); if (name === "afterEach" && suite?.suite) { await callSuiteHook(suite.suite, currentTask, name); diff --git a/packages/vitest-plugin/src/compat.ts b/packages/vitest-plugin/src/legacy/compat.ts similarity index 100% rename from packages/vitest-plugin/src/compat.ts rename to packages/vitest-plugin/src/legacy/compat.ts diff --git a/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts b/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts new file mode 100644 index 00000000..274db403 --- /dev/null +++ b/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts @@ -0,0 +1,46 @@ +// Vitest 3/4 exposed benchmark internals through `vitest/runners` and +// `vitest/suite`, and surfaced the `Benchmark` task type from `vitest`. Vitest 5 +// removed all of these (benchmarks now run through the unified `TestRunner`). +// +// The legacy runner (`analysis.ts`, `walltime/`) still imports them and is only +// loaded when the user runs Vitest 3/4, but the plugin is type-checked against +// whichever Vitest is installed — including 5, where these are gone. These +// ambient declarations keep the legacy code compiling there without affecting +// runtime: the modules are never imported under v5. + +import type * as tinybench from "tinybench"; + +declare module "vitest" { + import type { RunnerTestCase } from "vitest"; + + // In v3/4 a benchmark is a test case carrying tinybench output on its result. + interface Benchmark extends RunnerTestCase { + meta: RunnerTestCase["meta"] & { benchmark?: boolean }; + } +} + +declare module "vitest/runners" { + import type { RunnerTestSuite } from "vitest"; + + export class NodeBenchmarkRunner { + constructor(config?: unknown); + config: unknown; + runSuite(suite: RunnerTestSuite): Promise; + importTinybench(): Promise; + } +} + +declare module "vitest/suite" { + import type { Benchmark } from "vitest"; + + export function getBenchFn(benchmark: Benchmark): () => unknown; + export function getBenchOptions(benchmark: Benchmark): { + time?: number; + warmupTime?: number; + warmupIterations?: number; + iterations?: number; + }; + export function getHooks( + suite: unknown, + ): Record unknown>>; +} diff --git a/packages/vitest-plugin/src/legacy/walltime-utils.ts b/packages/vitest-plugin/src/legacy/walltime-utils.ts new file mode 100644 index 00000000..54bb4b1a --- /dev/null +++ b/packages/vitest-plugin/src/legacy/walltime-utils.ts @@ -0,0 +1,110 @@ +import { type Benchmark } from "@codspeed/core"; +import { + type RunnerTaskResult, + type RunnerTestSuite, + type Benchmark as VitestBenchmark, +} from "vitest"; +import { getBenchOptions } from "./compat"; +import { + tinybenchTaskToBenchmark, + type TinybenchOptions, + type TinybenchTask, +} from "../instrument"; +import { isVitestTaskBenchmark } from "./common"; + +export async function extractBenchmarkResults( + suite: RunnerTestSuite, + parentPath = "", +): Promise { + const benchmarks: Benchmark[] = []; + const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; + + for (const task of suite.tasks) { + if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { + const benchmark = processBenchmarkTask(task, currentPath); + if (benchmark) { + benchmarks.push(benchmark); + } + } else if (task.type === "suite") { + const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); + benchmarks.push(...nestedBenchmarks); + } + } + + return benchmarks; +} + +function processBenchmarkTask( + task: VitestBenchmark, + suitePath: string, +): Benchmark | null { + const uri = `${suitePath}::${task.name}`; + + const result = task.result; + if (!result) { + console.warn(` ⚠ No result data available for ${uri}`); + return null; + } + + try { + const benchOptions = getBenchOptions(task); + const benchmark = tinybenchTaskToBenchmark( + adaptLegacyResult(task.name, result), + uri, + benchOptions as TinybenchOptions, + ); + + if (benchmark === null) { + console.log(` ✔ No walltime data to collect for ${uri}`); + return null; + } + + console.log(` ✔ Collected walltime data for ${uri}`); + return benchmark; + } catch (error) { + console.warn(` ⚠ Failed to process benchmark result for ${uri}:`, error); + return null; + } +} + +/** + * Vitest 3/4 attaches the raw tinybench v2 result under `result.benchmark`, + * whose statistics are a flat object ({ totalTime, min, max, mean, sd, samples }). + * Reshape it into the `{ result: { totalTime, latency } }` form the shared + * converter expects (tinybench v6 nests statistics under `latency`). + */ +interface LegacyBenchmarkStats { + totalTime: number; + min: number; + max: number; + mean: number; + sd: number; + samples: number[]; +} + +function adaptLegacyResult( + name: string, + result: RunnerTaskResult, +): TinybenchTask { + // `result.benchmark` only exists on the Vitest 3/4 task result; the v5 typings + // (compiled against here) dropped it. + const benchmark = (result as { benchmark?: LegacyBenchmarkStats }).benchmark; + if (!benchmark) { + throw new Error("No benchmark data available in result"); + } + + return { + name, + fn: () => undefined, + result: { + totalTime: benchmark.totalTime, + latency: { + min: benchmark.min, + max: benchmark.max, + mean: benchmark.mean, + sd: benchmark.sd, + samples: benchmark.samples, + }, + }, + }; +} diff --git a/packages/vitest-plugin/src/legacy/walltime.ts b/packages/vitest-plugin/src/legacy/walltime.ts new file mode 100644 index 00000000..cd68eb84 --- /dev/null +++ b/packages/vitest-plugin/src/legacy/walltime.ts @@ -0,0 +1,119 @@ +import { setupCore } from "@codspeed/core"; +import type * as tinybench from "tinybench"; +import { + RunnerTaskEventPack, + RunnerTaskResultPack, + type RunnerTestSuite, +} from "vitest"; +import { NodeBenchmarkRunner } from "./compat"; +import { + installInstrumentHooks, + patchTaskRunOnce, + writeAndLogWalltimeResults, + type TinybenchBench, +} from "../instrument"; +import { patchRootSuiteWithFullFilePath } from "./common"; +import { extractBenchmarkResults } from "./walltime-utils"; + +type Tinybench = typeof tinybench; + +/** + * Lets tinybench run the benches through Vitest's default benchmark execution, + * instrumenting each measured loop, then extracts the results from the suite + * tree afterwards. (The v5 seam instruments the same way but reads results off + * the live tinybench tasks instead — see `v5/setup.ts`.) + */ +export class WalltimeRunner extends NodeBenchmarkRunner { + private suiteUris = new Map(); + /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks + private currentSuiteId: string | null = null; + + async runSuite(suite: RunnerTestSuite): Promise { + patchRootSuiteWithFullFilePath(suite); + this.populateBenchmarkUris(suite); + + setupCore(); + + await super.runSuite(suite); + + const benchmarks = await extractBenchmarkResults(suite); + if (benchmarks.length === 0) { + console.warn( + `[CodSpeed] No benchmark results found after suite execution`, + ); + return; + } + writeAndLogWalltimeResults(benchmarks); + } + + private populateBenchmarkUris(suite: RunnerTestSuite, parentPath = ""): void { + const currentPath = + parentPath !== "" ? `${parentPath}::${suite.name}` : suite.name; + + for (const task of suite.tasks) { + if (task.type === "suite") { + this.suiteUris.set(task.id, `${currentPath}::${task.name}`); + this.populateBenchmarkUris(task, currentPath); + } + } + } + + private getBenchmarkUri(taskName: string): string { + if (this.currentSuiteId === null) { + throw new Error("currentSuiteId is null - something went wrong"); + } + const suiteUri = this.suiteUris.get(this.currentSuiteId) || ""; + return `${suiteUri}::${taskName}`; + } + + async importTinybench(): Promise { + const tinybench = await super.importTinybench(); + + // `tinybench` is a frozen ES module namespace, so the `Bench` export cannot + // be reassigned. The shared `Task.prototype` is patched in place; the + // instrumented `Bench` is handed back through a fresh module-shaped object + // that Vitest destructures from. + patchTaskRunOnce(tinybench.Task); + + return { + ...tinybench, + Bench: this.createInstrumentedBench(tinybench), + }; + } + + private createInstrumentedBench( + tinybench: Tinybench, + ): typeof tinybench.Bench { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const runner = this; + const OriginalBench = tinybench.Bench; + + class InstrumentedBench extends OriginalBench { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(...benchArgs: any[]) { + super(...benchArgs); + installInstrumentHooks(this as unknown as TinybenchBench, (taskName) => + runner.getBenchmarkUri(taskName), + ); + } + } + + return InstrumentedBench; + } + + // Allow tinybench to retrieve the path to the currently running suite + async onTaskUpdate( + _: RunnerTaskResultPack[], + events: RunnerTaskEventPack[], + ): Promise { + events.map((event) => { + const [id, eventName] = event; + + if (eventName === "suite-prepare") { + this.currentSuiteId = id; + } + }); + } +} + +export default WalltimeRunner; diff --git a/packages/vitest-plugin/src/runner.ts b/packages/vitest-plugin/src/runner.ts deleted file mode 100644 index 60a627a6..00000000 --- a/packages/vitest-plugin/src/runner.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { AnalysisRunner } from "./analysis"; - -export default AnalysisRunner; diff --git a/packages/vitest-plugin/src/v5/setup.ts b/packages/vitest-plugin/src/v5/setup.ts new file mode 100644 index 00000000..6a851463 --- /dev/null +++ b/packages/vitest-plugin/src/v5/setup.ts @@ -0,0 +1,131 @@ +import { + getGitDir, + getInstrumentMode, + setupCore, + type Benchmark, +} from "@codspeed/core"; +import path from "path"; +// `TestRunner` is the unified runner Vitest 5 introduced; it replaces the +// `NodeBenchmarkRunner` the legacy seam subclasses, and is re-exported from the +// package root. It is read off the namespace (rather than a named import) so the +// plugin still type-checks against Vitest 3/4, which don't export it; this +// module only ever runs under v5. +import * as vitest from "vitest"; +import { + installInstrumentHooks, + patchTaskRunOnce, + tinybenchTaskToBenchmark, + writeAndLogWalltimeResults, + type TinybenchBench, + type TinybenchOptions, + type TinybenchTask, +} from "../instrument"; + +const TestRunner = (vitest as unknown as { TestRunner: unknown }).TestRunner; + +/** + * Vitest 5 runs benchmarks inside `test()` through `TestRunner`, calling the + * static `TestRunner.runBenchmarks(tinybench)` with a fully built tinybench + * instance. That static is referenced directly (not through `this`), so a + * runner subclass cannot intercept it — we patch the static in place instead. + * + * This module is registered as a Vitest `setupFile`, which runs in the worker + * before any test file is collected. Importing it installs the patch as a side + * effect. A setup file (rather than a custom `test.runner`) is used because it + * leaves the runner untouched for non-benchmark tests and also applies to the + * browser pool, where a Node runner file would not. + */ + +interface TinybenchWithTasks extends TinybenchBench { + name: string; + tasks: TinybenchTask[]; + run(): Promise; + // tinybench exposes the resolved options on the instance + opts?: TinybenchOptions; +} + +/** Minimal shape of the current Vitest test task we read for URI construction. */ +interface CurrentTest { + fullTestName?: string; + file?: { filepath: string }; +} + +function getCurrentTest(): CurrentTest | undefined { + // `getCurrentTest` is a static on the runner; typings don't surface it. + const getter = (TestRunner as unknown as { getCurrentTest?: () => unknown }) + .getCurrentTest; + return getter ? (getter() as CurrentTest | undefined) : undefined; +} + +/** + * Build the benchmark URI from the running test and the tinybench task name. + * Matches the legacy convention: git-relative file, then the suite/test path, + * then the bench name, all `::`-separated. + */ +function buildUri(taskName: string): string { + const test = getCurrentTest(); + const filepath = test?.file?.filepath; + if (!filepath) { + throw new Error("[CodSpeed] could not resolve the running benchmark file"); + } + const gitDir = getGitDir(filepath); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + const relativeFile = path.relative(gitDir, filepath); + // `fullTestName` uses " > " between suite levels; normalize to "::". + const testPath = (test?.fullTestName ?? "").split(" > ").join("::"); + return [relativeFile, testPath, taskName].filter(Boolean).join("::"); +} + +function collectWalltimeResults(tinybench: TinybenchWithTasks): void { + const options: TinybenchOptions = tinybench.opts ?? {}; + const benchmarks: Benchmark[] = []; + + for (const task of tinybench.tasks) { + if (task.result?.state && task.result.state !== "completed") continue; + const benchmark = tinybenchTaskToBenchmark( + task, + buildUri(task.name), + options, + ); + if (benchmark) { + benchmarks.push(benchmark); + } + } + + writeAndLogWalltimeResults(benchmarks); +} + +function patchRunBenchmarks(): void { + const Runner = TestRunner as unknown as { + runBenchmarks: (tinybench: TinybenchWithTasks) => Promise; + }; + const originalRunBenchmarks = Runner.runBenchmarks.bind(TestRunner); + const isWalltime = getInstrumentMode() === "walltime"; + + Runner.runBenchmarks = async (tinybench: TinybenchWithTasks) => { + setupCore(); + + // tinybench's `Task` class isn't exported from the bench instance, so we + // reach it through a constructed task (Vitest adds them before running). + const TaskClass = tinybench.tasks[0]?.constructor as + | { prototype: { run: (this: unknown) => Promise } } + | undefined; + if (TaskClass) { + patchTaskRunOnce(TaskClass); + } + + installInstrumentHooks(tinybench, buildUri); + + const result = await originalRunBenchmarks(tinybench); + + if (isWalltime) { + collectWalltimeResults(tinybench); + } + + return result; + }; +} + +patchRunBenchmarks(); diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts new file mode 100644 index 00000000..db8c0d26 --- /dev/null +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -0,0 +1,152 @@ +import { getInstrumentMode } from "@codspeed/core"; +import { readFileSync } from "fs"; +import { createRequire } from "module"; +import { join } from "path"; +import { type ViteUserConfig } from "vitest/config"; + +/** + * Everything about integrating with Vitest that depends on which Vitest + * generation the user installed, resolved once so the rest of the plugin reads a + * `VitestBackend` and never inspects the version itself. + */ +export interface VitestBackend { + /** + * Whether the plugin should stay active for this Vite `mode`. When false the + * plugin's `apply` returns false and it is dropped entirely. + */ + isActiveForViteMode(mode: string): boolean; + + /** + * Whether the current invocation is running benchmarks (as opposed to tests), + * given the incoming config and Vite `mode`. + */ + isBenchmarkRun(config: ViteUserConfig, mode: string): boolean; + + /** + * The `test` config fragment that wires the benchmark instrumentation into + * Vitest: the V8 exec args (whose placement moved across versions) plus the + * integration seam (a custom runner on legacy, a setup file on v5). + */ + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"]; + + /** + * The `test.benchmark` fragment asking tinybench to retain per-iteration + * samples so the walltime runner can compute quantiles. Only used in walltime + * mode. The option was renamed (`includeSamples` → `retainSamples`) when + * Vitest 5 moved to tinybench v6. + */ + getWalltimeBenchmarkConfig(): Record; +} + +/** + * Vitest 5 reworked the benchmark backend: the dedicated `NodeBenchmarkRunner` + * and the `vitest/runners` / `vitest/suite` entrypoints are gone, benchmarks run + * inside `test()` through the unified `TestRunner`, and tinybench moved to v6. + * The integration seam therefore differs fundamentally (a `TestRunner` patch + * installed from a setup file vs. a runner subclass per mode), which is why the + * two backends are separate implementations rather than a pile of inline + * ternaries. + * + * When the version cannot be detected we assume the latest supported major. + */ +export function resolveVitestBackend(): VitestBackend { + const major = getVitestMajorVersion() ?? 5; + return major >= 5 ? new V5Backend() : new LegacyBackend(major); +} + +/** + * Resolve the major version of the Vitest the *user's project* depends on, not + * the one bundled alongside this plugin. Returns null when it cannot be found, + * letting `resolveVitestBackend` fall back to the latest supported major. + */ +function getVitestMajorVersion(): number | null { + try { + const require = createRequire(join(process.cwd(), "package.json")); + const vitestPkgPath = require.resolve("vitest/package.json"); + const vitestPkg = JSON.parse(readFileSync(vitestPkgPath, "utf-8")); + return parseInt(vitestPkg.version.split(".")[0], 10); + } catch { + return null; + } +} + +/** + * Vitest 5+. `vitest bench` runs under the `"test"` mode with + * `test.benchmark.enabled` flipped (there is no dedicated benchmark mode), so + * the plugin stays active for every mode and gates on the config instead. + * Instrumentation is installed from a setup file that patches the shared + * `TestRunner` (see `v5/setup.ts`). + */ +class V5Backend implements VitestBackend { + isActiveForViteMode(): boolean { + return true; + } + + isBenchmarkRun(config: ViteUserConfig): boolean { + // `benchmark.enabled` only exists on the Vitest 5 config; the v3/4 typings + // we may be compiled against don't know about it. + const benchmark = config.test?.benchmark as + | { enabled?: boolean } + | undefined; + return benchmark?.enabled === true; + } + + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"] { + return { + execArgv: v8Flags, + setupFiles: [resolveFile("v5/setup")], + }; + } + + getWalltimeBenchmarkConfig(): Record { + return { retainSamples: true }; + } +} + +/** + * Vitest 3/4. `vitest bench` runs under a dedicated `"benchmark"` Vite mode, and + * instrumentation is installed through a custom `test.runner` subclass of + * `NodeBenchmarkRunner`, one per instrument mode (`analysis` / `walltime`). + */ +class LegacyBackend implements VitestBackend { + constructor(private readonly major: number) {} + + isActiveForViteMode(mode: string): boolean { + return mode === "benchmark"; + } + + isBenchmarkRun(_config: ViteUserConfig, mode: string): boolean { + return mode === "benchmark"; + } + + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"] { + const instrumentMode = getInstrumentMode(); + const runner = + instrumentMode === "disabled" + ? undefined + : resolveFile(join("legacy", instrumentMode)); + + return { + // Vitest 3 nests exec args under `poolOptions.forks`; v4 moved them to a + // top-level `test.execArgv`. + // See: https://vitest.dev/guide/migration.html#pool-rework + ...(this.major >= 4 + ? { execArgv: v8Flags } + : { poolOptions: { forks: { execArgv: v8Flags } } }), + ...(runner && { runner }), + }; + } + + getWalltimeBenchmarkConfig(): Record { + return { includeSamples: true }; + } +} diff --git a/packages/vitest-plugin/src/walltime/index.ts b/packages/vitest-plugin/src/walltime/index.ts deleted file mode 100644 index 880d9279..00000000 --- a/packages/vitest-plugin/src/walltime/index.ts +++ /dev/null @@ -1,216 +0,0 @@ -import { - InstrumentHooks, - MARKER_TYPE_BENCHMARK_END, - MARKER_TYPE_BENCHMARK_START, - setupCore, - wrapWithRootFrame, - writeWalltimeResults, -} from "@codspeed/core"; -import type * as tinybench from "tinybench"; -import { - RunnerTaskEventPack, - RunnerTaskResultPack, - type RunnerTestSuite, -} from "vitest"; -import { patchRootSuiteWithFullFilePath } from "../common"; -import { NodeBenchmarkRunner } from "../compat"; -import { extractBenchmarkResults } from "./utils"; - -type Tinybench = typeof tinybench; - -/** A tinybench task, exposing the `fn` the runner wraps with the root frame. */ -interface TinybenchTask { - name: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - fn: (...args: any[]) => any; -} - -/** tinybench's per-task setup/teardown hook signature. */ -type TinybenchHook = ( - task: TinybenchTask, - mode: "run" | "warmup", -) => Promise | void; - -/** The mutable subset of a tinybench Bench the runner reaches into. */ -interface TinybenchBench { - setup: TinybenchHook; - teardown: TinybenchHook; -} - -/** - * WalltimeRunner uses Vitest's default benchmark execution - * and extracts results from the suite after completion - */ -export class WalltimeRunner extends NodeBenchmarkRunner { - private isTinybenchHookedWithCodspeed = false; - private suiteUris = new Map(); - /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks - private currentSuiteId: string | null = null; - // Carries the window start timestamp from the setup hook to the teardown - // hook. Tasks run strictly sequentially, so a single field is enough. - private runStart: bigint | null = null; - - async runSuite(suite: RunnerTestSuite): Promise { - patchRootSuiteWithFullFilePath(suite); - this.populateBenchmarkUris(suite); - - setupCore(); - - await super.runSuite(suite); - - const benchmarks = await extractBenchmarkResults(suite); - - if (benchmarks.length > 0) { - writeWalltimeResults(benchmarks); - console.log( - `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.`, - ); - } else { - console.warn( - `[CodSpeed] No benchmark results found after suite execution`, - ); - } - } - - private populateBenchmarkUris(suite: RunnerTestSuite, parentPath = ""): void { - const currentPath = - parentPath !== "" ? `${parentPath}::${suite.name}` : suite.name; - - for (const task of suite.tasks) { - if (task.type === "suite") { - this.suiteUris.set(task.id, `${currentPath}::${task.name}`); - this.populateBenchmarkUris(task, currentPath); - } - } - } - - private getBenchmarkUri(taskName: string): string { - if (this.currentSuiteId === null) { - throw new Error("currentSuiteId is null - something went wrong"); - } - const suiteUri = this.suiteUris.get(this.currentSuiteId) || ""; - return `${suiteUri}::${taskName}`; - } - - async importTinybench(): Promise { - const tinybench = await super.importTinybench(); - - // `tinybench` is a frozen ES module namespace, so the `Bench` export cannot - // be reassigned. Mutating the shared `Task.prototype` in place is allowed - // and only needs to happen once; the instrumented `Bench` is handed back - // through a fresh module-shaped object that Vitest destructures from. - if (!this.isTinybenchHookedWithCodspeed) { - this.isTinybenchHookedWithCodspeed = true; - this.patchTaskWithRootFrame(tinybench); - } - - return { - ...tinybench, - Bench: this.createInstrumentedBench(tinybench), - }; - } - - /** - * Wrap each task's function with the root frame so collected stacks can be - * attributed to a benchmark. The window itself is driven by the bench's - * setup/teardown hooks (see createInstrumentedBench). - */ - private patchTaskWithRootFrame(tinybench: Tinybench): void { - const originalRun = tinybench.Task.prototype.run; - - tinybench.Task.prototype.run = async function () { - const task = this as unknown as TinybenchTask; - const originalFn = task.fn; - task.fn = wrapWithRootFrame(() => originalFn.call(task)); - - try { - await originalRun.call(this); - } finally { - task.fn = originalFn; - } - - return this; - }; - } - - /** - * Drive the instrumentation window from each bench's run-mode setup/teardown - * hooks so it brackets only tinybench's measured loop, excluding the warmup - * that Vitest runs beforehand and the statistics computation tinybench - * performs after the loop. Wrapping the whole `Task.run()` would otherwise - * fold all of that framework overhead into the recorded sample. - * - * User-provided hooks are preserved and keep their order relative to the work - * under test. - */ - private createInstrumentedBench( - tinybench: Tinybench, - ): typeof tinybench.Bench { - // eslint-disable-next-line @typescript-eslint/no-this-alias - const runner = this; - const OriginalBench = tinybench.Bench; - - class InstrumentedBench extends OriginalBench { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - constructor(...benchArgs: any[]) { - super(...benchArgs); - runner.installInstrumentHooks(this as unknown as TinybenchBench); - } - } - - return InstrumentedBench; - } - - private installInstrumentHooks(bench: TinybenchBench): void { - const userSetup = bench.setup; - const userTeardown = bench.teardown; - - bench.setup = async (task, mode) => { - await userSetup(task, mode); - if (mode === "run") { - InstrumentHooks.startBenchmark(); - this.runStart = InstrumentHooks.currentTimestamp(); - } - }; - - bench.teardown = async (task, mode) => { - if (mode === "run") { - this.closeInstrumentWindow(this.getBenchmarkUri(task.name)); - } - await userTeardown(task, mode); - }; - } - - private closeInstrumentWindow(uri: string): void { - const runEnd = InstrumentHooks.currentTimestamp(); - const pid = process.pid; - - // Benchmark markers must land inside the sample window opened by - // startBenchmark(), so they have to be emitted before stopBenchmark() - // closes it. The runner consumes the FIFO stream in order, so a marker - // sent after StopBenchmark falls outside the sample and breaks the - // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, this.runStart!); - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); - - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(pid, uri); - this.runStart = null; - } - - // Allow tinybench to retrieve the path to the currently running suite - async onTaskUpdate( - _: RunnerTaskResultPack[], - events: RunnerTaskEventPack[], - ): Promise { - events.map((event) => { - const [id, eventName] = event; - - if (eventName === "suite-prepare") { - this.currentSuiteId = id; - } - }); - } -} - -export default WalltimeRunner; diff --git a/packages/vitest-plugin/src/walltime/utils.ts b/packages/vitest-plugin/src/walltime/utils.ts deleted file mode 100644 index e0e3313e..00000000 --- a/packages/vitest-plugin/src/walltime/utils.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { - calculateQuantiles, - msToNs, - msToS, - type Benchmark, - type BenchmarkStats, -} from "@codspeed/core"; -import { - type RunnerTaskResult, - type RunnerTestSuite, - type Benchmark as VitestBenchmark, -} from "vitest"; -import { isVitestTaskBenchmark } from "../common"; -import { getBenchOptions } from "../compat"; - -export async function extractBenchmarkResults( - suite: RunnerTestSuite, - parentPath = "", -): Promise { - const benchmarks: Benchmark[] = []; - const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; - - for (const task of suite.tasks) { - if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { - const benchmark = await processBenchmarkTask(task, currentPath); - if (benchmark) { - benchmarks.push(benchmark); - } - } else if (task.type === "suite") { - const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); - benchmarks.push(...nestedBenchmarks); - } - } - - return benchmarks; -} - -async function processBenchmarkTask( - task: VitestBenchmark, - suitePath: string, -): Promise { - const uri = `${suitePath}::${task.name}`; - - const result = task.result; - if (!result) { - console.warn(` ⚠ No result data available for ${uri}`); - return null; - } - - try { - // Get tinybench configuration options from vitest - const benchOptions = getBenchOptions(task); - - const stats = convertVitestResultToBenchmarkStats(result, benchOptions); - - if (stats === null) { - console.log(` ✔ No walltime data to collect for ${uri}`); - return null; - } - - const coreBenchmark: Benchmark = { - name: task.name, - uri, - config: { - max_rounds: benchOptions.iterations ?? null, - max_time_ns: benchOptions.time ? msToNs(benchOptions.time) : null, - min_round_time_ns: null, // tinybench does not have an option for this - warmup_time_ns: - benchOptions.warmupIterations !== 0 && benchOptions.warmupTime - ? msToNs(benchOptions.warmupTime) - : null, - }, - stats, - }; - - console.log(` ✔ Collected walltime data for ${uri}`); - return coreBenchmark; - } catch (error) { - console.warn(` ⚠ Failed to process benchmark result for ${uri}:`, error); - return null; - } -} - -function convertVitestResultToBenchmarkStats( - result: RunnerTaskResult, - benchOptions: { - time?: number; - warmupTime?: number; - warmupIterations?: number; - iterations?: number; - }, -): BenchmarkStats | null { - const benchmark = result.benchmark; - - if (!benchmark) { - throw new Error("No benchmark data available in result"); - } - - const { totalTime, min, max, mean, sd, samples } = benchmark; - - // Get individual sample times in nanoseconds and sort them - const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); - const meanNs = msToNs(mean); - const stdevNs = msToNs(sd); - - if (sortedTimesNs.length == 0) { - // Sometimes the benchmarks can be completely optimized out and not even run, but its beforeEach and afterEach hooks are still executed, and the task is still considered a success. - // This is the case for the hooks.bench.ts example in this package - return null; - } - - const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = - calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); - - return { - min_ns: msToNs(min), - max_ns: msToNs(max), - mean_ns: meanNs, - stdev_ns: stdevNs, - q1_ns, - median_ns, - q3_ns, - total_time: msToS(totalTime), - iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration - rounds: sortedTimesNs.length, - iqr_outlier_rounds, - stdev_outlier_rounds, - warmup_iters: benchOptions.warmupIterations ?? 0, - }; -} diff --git a/packages/vitest-plugin/vitest.config.ts b/packages/vitest-plugin/vitest.config.ts index 672e3a27..444869bd 100644 --- a/packages/vitest-plugin/vitest.config.ts +++ b/packages/vitest-plugin/vitest.config.ts @@ -1,6 +1,18 @@ +import { createRequire } from "module"; import { defineConfig } from "vitest/config"; import codspeedPlugin from "./dist/index.mjs"; +// The legacy runner tests exercise the Vitest 3/4 benchmark backend +// (`vitest/suite`, `NodeBenchmarkRunner`), which Vitest 5 removed. Exclude them +// when running under v5+ so the file's static imports don't fail to resolve. +const require = createRequire(import.meta.url); +const vitestMajor = parseInt( + (require("vitest/package.json").version as string).split(".")[0], + 10, +); +const legacyOnlyTests = + vitestMajor >= 5 ? ["**/__tests__/instrumented.test.ts"] : []; + export default defineConfig({ // @ts-expect-error - TODO: investigate why importing from '.' wants to import only "main" field and thus fail plugins: [codspeedPlugin()], @@ -8,7 +20,7 @@ export default defineConfig({ __VERSION__: JSON.stringify("1.0.0"), }, test: { - exclude: ["**/tests/**/*", "**/.rollup.cache/**/*"], + exclude: ["**/tests/**/*", "**/.rollup.cache/**/*", ...legacyOnlyTests], mockReset: true, }, }); From b0a202be2f89f276acb5b3dc2198c6c3a0b0a07c Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 1 Jul 2026 11:04:44 +0200 Subject: [PATCH 2/6] test(vitest-plugin): add with-vitest-v4 example Now that the plugin's own dev dependency tracks Vitest 5, add a dedicated Vitest 4 example so the legacy (v3/4) benchmark seam keeps explicit coverage alongside the existing with-vitest-v3 example. Mirrors that example, pinning vitest ^4.1.9. Refs COD-2931 Co-Authored-By: Claude --- examples/with-vitest-v4/package.json | 13 ++++++++++++ .../with-vitest-v4/src/fibonacci.bench.ts | 20 +++++++++++++++++++ examples/with-vitest-v4/src/fibonacci.ts | 17 ++++++++++++++++ examples/with-vitest-v4/tsconfig.json | 13 ++++++++++++ examples/with-vitest-v4/vitest.config.ts | 6 ++++++ 5 files changed, 69 insertions(+) create mode 100644 examples/with-vitest-v4/package.json create mode 100644 examples/with-vitest-v4/src/fibonacci.bench.ts create mode 100644 examples/with-vitest-v4/src/fibonacci.ts create mode 100644 examples/with-vitest-v4/tsconfig.json create mode 100644 examples/with-vitest-v4/vitest.config.ts diff --git a/examples/with-vitest-v4/package.json b/examples/with-vitest-v4/package.json new file mode 100644 index 00000000..8a67c417 --- /dev/null +++ b/examples/with-vitest-v4/package.json @@ -0,0 +1,13 @@ +{ + "name": "with-vitest-v4", + "private": true, + "type": "module", + "scripts": { + "bench-vitest": "vitest bench --run" + }, + "devDependencies": { + "@codspeed/vitest-plugin": "workspace:*", + "typescript": "^5.1.3", + "vitest": "^4.1.9" + } +} diff --git a/examples/with-vitest-v4/src/fibonacci.bench.ts b/examples/with-vitest-v4/src/fibonacci.bench.ts new file mode 100644 index 00000000..227a6e67 --- /dev/null +++ b/examples/with-vitest-v4/src/fibonacci.bench.ts @@ -0,0 +1,20 @@ +import { bench, describe } from "vitest"; +import { iterativeFibonacci, recursiveFibonacci } from "./fibonacci"; + +describe("fibonacci", () => { + bench("recursive fibo 15", () => { + recursiveFibonacci(15); + }); + + bench("recursive fibo 20", () => { + recursiveFibonacci(20); + }); + + bench("iterative fibo 15", () => { + iterativeFibonacci(15); + }); + + bench("iterative fibo 20", () => { + iterativeFibonacci(20); + }); +}); diff --git a/examples/with-vitest-v4/src/fibonacci.ts b/examples/with-vitest-v4/src/fibonacci.ts new file mode 100644 index 00000000..94796660 --- /dev/null +++ b/examples/with-vitest-v4/src/fibonacci.ts @@ -0,0 +1,17 @@ +export function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +export function iterativeFibonacci(n: number): number { + let a = 0; + let b = 1; + for (let i = 0; i < n; i++) { + const temp = a + b; + a = b; + b = temp; + } + return a; +} diff --git a/examples/with-vitest-v4/tsconfig.json b/examples/with-vitest-v4/tsconfig.json new file mode 100644 index 00000000..ace1de03 --- /dev/null +++ b/examples/with-vitest-v4/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "verbatimModuleSyntax": true, + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/examples/with-vitest-v4/vitest.config.ts b/examples/with-vitest-v4/vitest.config.ts new file mode 100644 index 00000000..4b1290c1 --- /dev/null +++ b/examples/with-vitest-v4/vitest.config.ts @@ -0,0 +1,6 @@ +import codspeedPlugin from "@codspeed/vitest-plugin"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [codspeedPlugin()], +}); From 29ab3b98625f72b1c450414ea11df25a5af1b37a Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 4 Sep 2026 12:33:06 +0200 Subject: [PATCH 3/6] docs: document how to exercise a plugin in the three CodSpeed modes A plugin behaves differently depending on whether CodSpeed drives the run, and each mode takes a different path through the plugin, so all three have to be run when developing or reviewing one. Write down the commands and the expected output per mode, and ignore the local `.codspeed` result directory those runs produce. Co-Authored-By: Claude --- .gitignore | 2 ++ CLAUDE.md | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5233f61e..3cfb1201 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,5 @@ packages/app/.env # turbo .turbo/ .rollup.cache/ + +.codspeed diff --git a/CLAUDE.md b/CLAUDE.md index b82671a6..00a76a53 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,4 +94,25 @@ Based on the codebase analysis, to add stats access features: - Build: `pnpm turbo run build --filter=` - Typecheck: `pnpm turbo run typecheck --filter=` - Lint: `pnpm turbo run lint --filter=` - - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`). \ No newline at end of file + - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`). + +## Testing a plugin during development + +A plugin behaves differently depending on whether CodSpeed is driving the run, +so exercise all three of these when developing or reviewing a plugin change +(build the plugin first — the benches import from `dist`): + +1. **Fallback (not under CodSpeed).** No env vars. The plugin must stay out of + the way and let the framework run its benchmarks normally (no instrumentation, + no hijacked output). e.g. `pnpm turbo run bench --filter=`. +2. **Instrumentation / simulation.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=simulation` + (or `instrumentation`). The plugin hijacks the run to do a single instrumented + pass per benchmark and prints `Measured/Checked ` instead of the normal + harness output. +3. **Walltime.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=walltime`. The plugin + instruments the framework's real benchmark loop and collects walltime results. + +Running these locally outside the CodSpeed runner is expected to log +`instrument-hooks: failed to write environment.json` and skip actual measurement +writes — the point is to verify the plugin's control flow and output per mode, +not to produce real measurements. \ No newline at end of file From 1595df25724a185c472cc452365961ff753a5eee Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 4 Sep 2026 12:33:36 +0200 Subject: [PATCH 4/6] feat(vitest-plugin): use the vitest 5 benchmark provider API Vitest 5 replaced the top-level `bench()` declaration and its `NodeBenchmarkRunner` with a `bench` test-context fixture executed by a `benchmark.provider`: a module Vitest hands the registered functions, their options and the test that declared them, and whose results it treats as authoritative. Wire the V5 backend to register CodSpeed as that provider and add the provider itself, which runs the analysis and walltime modes off the registrations it receives. Benchmark URIs now carry the registration name after the test path, so a test declaring several benchmarks through `bench.compare()` reports one benchmark per registration. Move the package's own benches to the fixture API to cover both shapes, and read imported bindings into a local before the measured loop so the module-runner getters stay out of the samples. Gate the injection on the instrument mode: Vitest 5 clones its benchmark project after the Vite config hooks ran, so a benchmark run can no longer be detected from the incoming config, and CodSpeed only ever drives benchmark runs. Without CodSpeed the plugin injects nothing and Vitest runs the benchmarks through its own tinybench provider. Vitest 5 also runs `globalSetup` once per project, including the benchmark project it clones, so make setup and teardown idempotent instead of failing the run on the second teardown. The legacy (Vitest 3/4) runner keeps working: its Vitest lookups move behind `legacy/compat`, which now owns the types of the removed benchmark backend so the package type-checks against Vitest 5. Refs COD-2931 Co-Authored-By: Claude --- packages/vitest-plugin/benches/flat.bench.ts | 37 ++- packages/vitest-plugin/benches/hooks.bench.ts | 54 ++-- packages/vitest-plugin/benches/macos.bench.ts | 8 +- .../vitest-plugin/benches/parsePr.bench.ts | 58 ++-- .../vitest-plugin/benches/timing.bench.ts | 41 ++- packages/vitest-plugin/rollup.config.mjs | 4 +- .../src/__tests__/globalSetup.test.ts | 12 +- .../vitest-plugin/src/__tests__/index.test.ts | 31 +- .../src/__tests__/instrumented.test.ts | 19 +- packages/vitest-plugin/src/globalSetup.ts | 12 +- packages/vitest-plugin/src/index.ts | 4 - packages/vitest-plugin/src/instrument.ts | 174 +++++++++--- packages/vitest-plugin/src/legacy/analysis.ts | 17 +- packages/vitest-plugin/src/legacy/common.ts | 6 +- packages/vitest-plugin/src/legacy/compat.ts | 68 ++++- .../src/legacy/vitest-legacy.d.ts | 51 +--- .../src/legacy/walltime-utils.ts | 18 +- packages/vitest-plugin/src/legacy/walltime.ts | 6 +- packages/vitest-plugin/src/v5/provider.ts | 267 ++++++++++++++++++ packages/vitest-plugin/src/v5/setup.ts | 131 --------- packages/vitest-plugin/src/vitestBackend.ts | 70 +++-- packages/vitest-plugin/vitest.config.ts | 14 +- pnpm-lock.yaml | 160 ++++++++++- 23 files changed, 843 insertions(+), 419 deletions(-) create mode 100644 packages/vitest-plugin/src/v5/provider.ts delete mode 100644 packages/vitest-plugin/src/v5/setup.ts diff --git a/packages/vitest-plugin/benches/flat.bench.ts b/packages/vitest-plugin/benches/flat.bench.ts index 74e67b9d..b9b494a4 100644 --- a/packages/vitest-plugin/benches/flat.bench.ts +++ b/packages/vitest-plugin/benches/flat.bench.ts @@ -1,5 +1,10 @@ -import { bench, describe } from "vitest"; -import parsePr from "./parsePr"; +import { describe, test } from "vitest"; +import parsePrExport from "./parsePr"; + +// Read the imported binding once: Vite's module runner exposes exports through +// getters, and reading one inside the measured loop adds overhead to every +// iteration (Vitest warns about it). +const parsePr = parsePrExport; const LONG_BODY = new Array(1_000) @@ -9,12 +14,15 @@ const LONG_BODY = .join("\n") + "fixes #123"; describe("parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test-1", number: 1 }); - }); - - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test-2", number: 2 }); + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", () => { + parsePr({ body: "fixes #123", title: "test-1", number: 1 }); + }), + bench("long body", () => { + parsePr({ body: LONG_BODY, title: "test-2", number: 2 }); + }), + ); }); }); @@ -24,10 +32,13 @@ function fibo(n: number): number { } describe("fibo", () => { - bench("fibo 10", () => { - fibo(10); - }); - bench("fibo 15", () => { - fibo(15); + test("depth", async ({ bench }) => { + await bench("fibo 10", () => { + fibo(10); + }).run(); + + await bench("fibo 15", () => { + fibo(15); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/hooks.bench.ts b/packages/vitest-plugin/benches/hooks.bench.ts index d790ca3c..c706dda8 100644 --- a/packages/vitest-plugin/benches/hooks.bench.ts +++ b/packages/vitest-plugin/benches/hooks.bench.ts @@ -1,37 +1,39 @@ -import { - afterAll, - afterEach, - beforeAll, - beforeEach, - bench, - describe, - expect, -} from "vitest"; +import { describe, expect, test } from "vitest"; +// Exercises tinybench's per-benchmark hooks, which Vitest 5 exposes through the +// `bench(name, options, fn)` options object (`beforeAll`/`beforeEach`/...). describe("hooks", () => { let count = 0; - describe("run", () => { - beforeAll(() => { + + const hooks = { + beforeAll: () => { count += 10; - }); - beforeEach(() => { + }, + beforeEach: () => { count += 1; - }); - afterEach(() => { + }, + afterEach: () => { count -= 1; - }); - afterAll(() => { + }, + afterAll: () => { count -= 10; - }); + }, + }; - bench("one", () => { - expect(count).toBe(11); - }); - bench("two", () => { - expect(count).toBe(11); - }); + test("hooked benches", async ({ bench }) => { + await bench.compare( + bench("one", hooks, () => { + expect(count).toBe(11); + }), + bench("two", hooks, () => { + expect(count).toBe(11); + }), + ); }); - bench("end", () => { - expect(count).toBe(0); + + test("after the hooked benches", async ({ bench }) => { + await bench("count is back to zero", () => { + expect(count).toBe(0); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/macos.bench.ts b/packages/vitest-plugin/benches/macos.bench.ts index e05a3082..792e9c3f 100644 --- a/packages/vitest-plugin/benches/macos.bench.ts +++ b/packages/vitest-plugin/benches/macos.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe } from "vitest"; +import { describe, test } from "vitest"; const isMacOS = process.platform === "darwin"; @@ -10,7 +10,9 @@ function fibo(n: number): number { // macOS-only benchmark: skipped on every other platform, so it only runs on // the `codspeed-walltime-macos` CI job (see .github/workflows/codspeed.yml). describe.skipIf(!isMacOS)("macos only", () => { - bench("fibo darwin", () => { - fibo(30); + test("fibo", async ({ bench }) => { + await bench("fibo darwin", () => { + fibo(30); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/parsePr.bench.ts b/packages/vitest-plugin/benches/parsePr.bench.ts index 91c0adee..60829edf 100644 --- a/packages/vitest-plugin/benches/parsePr.bench.ts +++ b/packages/vitest-plugin/benches/parsePr.bench.ts @@ -1,5 +1,10 @@ -import { bench, describe } from "vitest"; -import parsePr from "./parsePr"; +import { describe, test } from "vitest"; +import parsePrExport from "./parsePr"; + +// Read the imported binding once: Vite's module runner exposes exports through +// getters, and reading one inside the measured loop adds overhead to every +// iteration (Vitest warns about it). +const parsePr = parsePrExport; const LONG_BODY = new Array(1_000) @@ -8,44 +13,49 @@ const LONG_BODY = ) .join("\n") + "fixes #123"; -describe("parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); - }); +function benchShortBody() { + parsePr({ body: "fixes #123", title: "test", number: 124 }); +} - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); +function benchLongBody() { + parsePr({ body: LONG_BODY, title: "test", number: 124 }); +} + +describe("parsePr", () => { + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", benchShortBody), + bench("long body", benchLongBody), + ); }); describe("nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); - }); - - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", benchShortBody), + bench("long body", benchLongBody), + ); }); describe("deeply nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); }); }); }); describe("another parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); - }); - - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", benchShortBody), + bench("long body", benchLongBody), + ); }); describe("nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); }); }); diff --git a/packages/vitest-plugin/benches/timing.bench.ts b/packages/vitest-plugin/benches/timing.bench.ts index 83331ad6..fd6aa5f8 100644 --- a/packages/vitest-plugin/benches/timing.bench.ts +++ b/packages/vitest-plugin/benches/timing.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe, type BenchOptions } from "vitest"; +import { describe, test, type BenchCompareOptions } from "vitest"; const busySleep = (ms: number): void => { const end = performance.now() + ms; @@ -7,33 +7,24 @@ const busySleep = (ms: number): void => { } }; -const timingBenchOptions: BenchOptions = { +const timingBenchOptions: BenchCompareOptions = { iterations: 5, warmupIterations: 0, }; describe("timing tests", () => { - bench( - "wait 1ms", - async () => { - busySleep(1); - }, - timingBenchOptions, - ); - - bench( - "wait 500ms", - async () => { - busySleep(500); - }, - timingBenchOptions, - ); - - bench( - "wait 1sec", - async () => { - busySleep(1_000); - }, - timingBenchOptions, - ); + test("busy sleep", async ({ bench }) => { + await bench.compare( + bench("wait 1ms", async () => { + busySleep(1); + }), + bench("wait 500ms", async () => { + busySleep(500); + }), + bench("wait 1sec", async () => { + busySleep(1_000); + }), + timingBenchOptions, + ); + }); }); diff --git a/packages/vitest-plugin/rollup.config.mjs b/packages/vitest-plugin/rollup.config.mjs index 6c9827c7..0c1a0948 100644 --- a/packages/vitest-plugin/rollup.config.mjs +++ b/packages/vitest-plugin/rollup.config.mjs @@ -40,8 +40,8 @@ export default defineConfig([ external: ["@codspeed/core", /^vitest/], }, { - input: "src/v5/setup.ts", - output: { file: "dist/v5/setup.mjs", format: "es" }, + input: "src/v5/provider.ts", + output: { file: "dist/v5/provider.mjs", format: "es" }, plugins: jsPlugins(pkg.version), external: ["@codspeed/core", /^vitest/, "tinybench"], }, diff --git a/packages/vitest-plugin/src/__tests__/globalSetup.test.ts b/packages/vitest-plugin/src/__tests__/globalSetup.test.ts index 178b3ecb..f475f048 100644 --- a/packages/vitest-plugin/src/__tests__/globalSetup.test.ts +++ b/packages/vitest-plugin/src/__tests__/globalSetup.test.ts @@ -4,19 +4,27 @@ import globalSetup from "../globalSetup"; console.log = vi.fn(); describe("globalSetup", () => { - it("should log the correct message on setup and teardown, and fail when teardown is called twice", async () => { + it("should log setup and teardown once, even when Vitest runs them per project", async () => { const teardown = globalSetup(); expect(console.log).toHaveBeenCalledWith( "[CodSpeed] @codspeed/vitest-plugin v1.0.0 - setup", ); + // Vitest 5 runs the same globalSetup for the base project and for the + // benchmark project it clones from it. + globalSetup(); + + expect(console.log).toHaveBeenCalledTimes(1); + teardown(); expect(console.log).toHaveBeenCalledWith( "[CodSpeed] @codspeed/vitest-plugin v1.0.0 - teardown", ); - expect(() => teardown()).toThrowError("teardown called twice"); + teardown(); + + expect(console.log).toHaveBeenCalledTimes(2); }); }); diff --git a/packages/vitest-plugin/src/__tests__/index.test.ts b/packages/vitest-plugin/src/__tests__/index.test.ts index f129c229..da284190 100644 --- a/packages/vitest-plugin/src/__tests__/index.test.ts +++ b/packages/vitest-plugin/src/__tests__/index.test.ts @@ -1,5 +1,5 @@ -import { fromPartial } from "@total-typescript/shoehorn"; import { getV8Flags } from "@codspeed/core"; +import { fromPartial } from "@total-typescript/shoehorn"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import codspeedPlugin from "../index"; @@ -106,7 +106,7 @@ describe("codSpeedPlugin", () => { }); it("should stay active regardless of mode on v5 (benchmark gating happens in config)", async () => { - fsMocks.setMockVersion("5.0.0-beta.5"); + fsMocks.setMockVersion("5.0.0"); coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true); const applyPlugin = applyPluginFunction( @@ -180,8 +180,10 @@ describe("codSpeedPlugin", () => { }); describe("v5 config", () => { - it("should not inject config when benchmarks are not enabled", () => { - fsMocks.setMockVersion("5.0.0-beta.5"); + it("should not inject config when CodSpeed is not driving the run", () => { + fsMocks.setMockVersion("5.0.0"); + delete process.env.CODSPEED_ENV; + const v5Plugin = codspeedPlugin(); const config = v5Plugin.config; if (typeof config !== "function") @@ -194,11 +196,12 @@ describe("codSpeedPlugin", () => { ); expect(result).toBeUndefined(); + process.env.CODSPEED_ENV = "1"; fsMocks.setMockVersion("4.0.18"); }); - it("should inject the v5 setup file (not a runner) when benchmarks are enabled", () => { - fsMocks.setMockVersion("5.0.0-beta.5"); + it("should wire the v5 benchmark provider (not a runner or setup file)", () => { + fsMocks.setMockVersion("5.0.0"); const v5Plugin = codspeedPlugin(); const config = v5Plugin.config; if (typeof config !== "function") @@ -206,9 +209,7 @@ describe("codSpeedPlugin", () => { const result = config.call( {} as never, - // `benchmark.enabled` is a Vitest 5 config field the v3/4 typings (which - // this file may be compiled against) don't expose. - { test: { benchmark: { enabled: true } } } as never, + {}, fromPartial({ mode: "test" }), ); @@ -221,15 +222,13 @@ describe("codSpeedPlugin", () => { ], pool: "forks", execArgv: getV8Flags(), - setupFiles: [ - expect.stringContaining("packages/vitest-plugin/src/v5/setup.ts"), - ], + benchmark: { + provider: expect.stringContaining( + "packages/vitest-plugin/src/v5/provider.ts", + ), + }, }, }); - // The v5 path must not set a custom runner. - expect( - (result as { test?: { runner?: unknown } })?.test?.runner, - ).toBeUndefined(); fsMocks.setMockVersion("4.0.18"); }); }); diff --git a/packages/vitest-plugin/src/__tests__/instrumented.test.ts b/packages/vitest-plugin/src/__tests__/instrumented.test.ts index a3106544..c3e9178a 100644 --- a/packages/vitest-plugin/src/__tests__/instrumented.test.ts +++ b/packages/vitest-plugin/src/__tests__/instrumented.test.ts @@ -3,10 +3,9 @@ import { describe, expect, it, vi, type RunnerTestSuite } from "vitest"; import { AnalysisRunner as CodSpeedRunner } from "../legacy/analysis"; import { getBenchFn, getBenchOptions } from "../legacy/compat"; -// The legacy AnalysisRunner targets the Vitest 3/4 benchmark backend -// (`NodeBenchmarkRunner`, `vitest/suite`), which Vitest 5 removed. This whole -// file is excluded from the test run under v5+ (see vitest.config.ts); the v5 -// path is covered separately. +// `legacy/compat` resolves the Vitest 3/4 benchmark backend from the installed +// Vitest, so it is mocked entirely: that backend is gone on Vitest 5, which this +// package is developed against. const coreMocks = vi.hoisted(() => { return { @@ -33,10 +32,16 @@ vi.mock("@codspeed/core", async (importOriginal) => { console.log = vi.fn(); -vi.mock("../legacy/compat", async (importOriginal) => { - const actual = await importOriginal(); +vi.mock("../legacy/compat", () => { + class NodeBenchmarkRunner { + async importTinybench() { + return import("tinybench"); + } + } + return { - ...actual, + NodeBenchmarkRunner, + getHooks: vi.fn(), getBenchFn: vi.fn(), getBenchOptions: vi.fn(), }; diff --git a/packages/vitest-plugin/src/globalSetup.ts b/packages/vitest-plugin/src/globalSetup.ts index 8c5e1a34..143dafea 100644 --- a/packages/vitest-plugin/src/globalSetup.ts +++ b/packages/vitest-plugin/src/globalSetup.ts @@ -9,13 +9,21 @@ function logCodSpeed(message: string) { console.log(`[CodSpeed] ${message}`); } +let setupHappened = false; let teardownHappened = false; +// Vitest 5 clones a dedicated `(bench)` project from the base one and runs the +// same globalSetup module for both, so setup and teardown each fire twice. +// Report only the first pass and make the repeats no-ops; throwing on the second +// teardown fails the whole run during close. export default function () { - logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`); + if (!setupHappened) { + setupHappened = true; + logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`); + } return () => { - if (teardownHappened) throw new Error("teardown called twice"); + if (teardownHappened) return; teardownHappened = true; logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - teardown`); diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index c3216cab..cbbe5be0 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -1,5 +1,4 @@ import { - getCodspeedRunnerMode, getInstrumentMode, getV8Flags, InstrumentHooks, @@ -56,9 +55,6 @@ export default function codspeedPlugin(): Plugin { pool: "forks", globalSetup: [resolveFile("globalSetup")], ...backend.getBenchmarkTestConfig(getV8Flags(), resolveFile), - ...(getCodspeedRunnerMode() === "walltime" && { - benchmark: backend.getWalltimeBenchmarkConfig(), - }), }, }; diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts index b880ca89..cd069946 100644 --- a/packages/vitest-plugin/src/instrument.ts +++ b/packages/vitest-plugin/src/instrument.ts @@ -5,7 +5,10 @@ import { MARKER_TYPE_BENCHMARK_START, msToNs, msToS, + optimizeFunction, + optimizeFunctionSync, wrapWithRootFrame, + wrapWithRootFrameSync, writeWalltimeResults, type Benchmark, type BenchmarkStats, @@ -14,6 +17,21 @@ import type * as tinybench from "tinybench"; export type Tinybench = typeof tinybench; +/** tinybench's per-task lifecycle hooks (a subset of `FnOptions`). */ +export interface TinybenchFnOptions { + beforeAll?: (mode?: "run" | "warmup") => unknown; + beforeEach?: (mode?: "run" | "warmup") => unknown; + afterEach?: (mode?: "run" | "warmup") => unknown; + afterAll?: (mode?: "run" | "warmup") => unknown; +} + +/** The captured registration for a task: its fn and options. */ +export interface CapturedTask { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fn: (...args: any[]) => any; + fnOpts?: TinybenchFnOptions; +} + /** A tinybench task, exposing the `fn` the runner wraps with the root frame. */ export interface TinybenchTask { name: string; @@ -34,18 +52,6 @@ export interface TinybenchBench { teardown: TinybenchHook; } -/** The minimal task shape `patchTaskRunWithRootFrame` mutates. */ -interface RunnableTask { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - fn: (...args: any[]) => any; -} - -/** The tinybench Task prototype whose `run` we wrap. */ -interface TinybenchTaskClass { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - prototype: { run: (this: any) => Promise }; -} - /** * The tinybench statistics shape (latency/throughput) shared across the v2 and * v6 lines. Only the fields the conversion needs are modeled. @@ -60,6 +66,8 @@ interface TinybenchStatistics { interface TinybenchTaskResult { state?: string; + /** Set by tinybench when `state` is `"errored"`. */ + error?: Error; totalTime: number; latency: TinybenchStatistics; } @@ -77,8 +85,6 @@ interface InstrumentWindow { runStart: bigint | null; } -let isTaskPatched = false; - /** * The window bracketing the currently running task's measured loop, driven by * the setup/teardown hooks below. Tasks run strictly sequentially within a @@ -86,14 +92,22 @@ let isTaskPatched = false; */ const instrumentWindow: InstrumentWindow = { runStart: null }; +/** The tinybench Task prototype whose `run` the legacy seam wraps. */ +interface TinybenchTaskClass { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prototype: { run: (this: any) => Promise }; +} + +let isTaskPatched = false; + /** - * Wrap every task's fn with the root frame so collected stacks are attributed to - * a benchmark. Idempotent: patching the shared `Task.prototype.run` in place hits - * every Bench instance, so repeat calls are no-ops. + * Wrap every task's fn with the root frame by patching `Task.prototype.run` in + * place. Used only by the legacy (Vitest 3/4) walltime seam, which runs on + * tinybench v2 where a task's `fn` is a plain, reassignable property. * - * `TaskClass` must be the exact prototype the host constructed its tasks against - * (taken from a live task, not imported) so the patch applies even when multiple - * copies of tinybench are installed. + * The Vitest 5 seam cannot use this: tinybench v6 made `fn` a true `#private` + * field, so reassigning `task.fn` is a silent no-op there — the frame must be + * baked in at registration time instead (see rootFrameRegisterFn). */ export function patchTaskRunOnce(TaskClass: TinybenchTaskClass): void { if (isTaskPatched) { @@ -102,7 +116,7 @@ export function patchTaskRunOnce(TaskClass: TinybenchTaskClass): void { isTaskPatched = true; const originalRun = TaskClass.prototype.run; - TaskClass.prototype.run = async function (this: RunnableTask) { + TaskClass.prototype.run = async function (this: CapturedTask) { const originalFn = this.fn; this.fn = wrapWithRootFrame(() => originalFn.call(this)); @@ -114,6 +128,93 @@ export function patchTaskRunOnce(TaskClass: TinybenchTaskClass): void { }; } +/** + * The root-frame wrap to hand tinybench at registration time. Post-hoc + * assignment to a task's `fn` is a no-op on tinybench v6 (private field), so the + * frame must be baked into the registered fn instead. + */ +export function rootFrameRegisterFn( + fn: CapturedTask["fn"], +): CapturedTask["fn"] { + return wrapWithRootFrame(() => fn()); +} + +/** + * Run one benchmark under instrumentation, matching the analysis window the + * Vitest 3/4 runner uses exactly: warm the JIT with `optimizeFunction` outside + * the window, run the user hooks around a single measured `fn()`, and bracket + * only that call with `startBenchmark`/`stopBenchmark` under the root frame. The + * measurement comes from the instrument, so no wall-clock markers are emitted + * and tinybench's timing loop is not involved. + * + * Synchronous benchmarks run through a fully synchronous window + * (`wrapWithRootFrameSync`, no `await`): awaiting a sync fn would splice Node's + * promise-hook machinery in above the root frame and pollute the sample. Async + * benchmarks necessarily use the awaited path. + */ +export async function runAnalysisTask( + { fn, fnOpts }: CapturedTask, + uri: string, +): Promise { + if (isAsyncFn(fn)) { + await runAnalysisTaskAsync(fn, fnOpts, uri); + } else { + await runAnalysisTaskSync(fn, fnOpts, uri); + } +} + +function isAsyncFn(fn: CapturedTask["fn"]): boolean { + return fn.constructor?.name === "AsyncFunction"; +} + +async function runAnalysisTaskAsync( + fn: CapturedTask["fn"], + fnOpts: TinybenchFnOptions | undefined, + uri: string, +): Promise { + await fnOpts?.beforeAll?.("run"); + await optimizeFunction(async () => { + await fnOpts?.beforeEach?.("run"); + await fn(); + await fnOpts?.afterEach?.("run"); + }); + + await fnOpts?.beforeEach?.("run"); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await fnOpts?.afterEach?.("run"); + await fnOpts?.afterAll?.("run"); +} + +function runAnalysisTaskSync( + fn: CapturedTask["fn"], + fnOpts: TinybenchFnOptions | undefined, + uri: string, +): void { + fnOpts?.beforeAll?.("run"); + optimizeFunctionSync(() => { + fnOpts?.beforeEach?.("run"); + fn(); + fnOpts?.afterEach?.("run"); + }); + + fnOpts?.beforeEach?.("run"); + global.gc?.(); + wrapWithRootFrameSync(() => { + InstrumentHooks.startBenchmark(); + fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + fnOpts?.afterEach?.("run"); + fnOpts?.afterAll?.("run"); +} + /** * Drive the instrumentation window from each bench's run-mode setup/teardown * hooks so it brackets only tinybench's measured loop, excluding the warmup @@ -148,24 +249,29 @@ export function installInstrumentHooks( } function closeInstrumentWindow(uri: string): void { - const runEnd = InstrumentHooks.currentTimestamp(); + emitBenchmarkWindow(uri, instrumentWindow.runStart!); + instrumentWindow.runStart = null; +} + +/** + * Close the currently open instrumentation window: emit the benchmark markers + * bracketing [start, now], stop the benchmark, and attribute the sample to `uri`. + * + * Benchmark markers must land inside the sample window opened by + * startBenchmark(), so they are emitted before stopBenchmark() closes it. The + * runner consumes the FIFO stream in order, so a marker sent after stopBenchmark + * would fall outside the sample and break the expected + * SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. + */ +function emitBenchmarkWindow(uri: string, start: bigint): void { + const end = InstrumentHooks.currentTimestamp(); const pid = process.pid; - // Benchmark markers must land inside the sample window opened by - // startBenchmark(), so they have to be emitted before stopBenchmark() - // closes it. The runner consumes the FIFO stream in order, so a marker - // sent after StopBenchmark falls outside the sample and breaks the - // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. - InstrumentHooks.addMarker( - pid, - MARKER_TYPE_BENCHMARK_START, - instrumentWindow.runStart!, - ); - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, start); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, end); InstrumentHooks.stopBenchmark(); InstrumentHooks.setExecutedBenchmark(pid, uri); - instrumentWindow.runStart = null; } /** diff --git a/packages/vitest-plugin/src/legacy/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts index 9821003e..336e2392 100644 --- a/packages/vitest-plugin/src/legacy/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -8,14 +8,19 @@ import { wrapWithRootFrame, } from "@codspeed/core"; import type * as tinybench from "tinybench"; -import { Benchmark, type RunnerTestSuite } from "vitest"; +import { type RunnerTestSuite } from "vitest"; import { callSuiteHook, isVitestTaskBenchmark, patchRootSuiteWithFullFilePath, } from "./common"; -import { getBenchFn, getBenchOptions, NodeBenchmarkRunner } from "./compat"; +import { + getBenchFn, + getBenchOptions, + NodeBenchmarkRunner, + type BenchmarkTask, +} from "./compat"; type Tinybench = typeof tinybench; @@ -34,7 +39,7 @@ function logCodSpeed(message: string) { } async function runAnalysisBench( - benchmark: Benchmark, + benchmark: BenchmarkTask, suite: RunnerTestSuite, currentSuiteName: string, tinybenchModule: Tinybench, @@ -91,10 +96,10 @@ async function runAnalysisBenchmarkSuite( for (const task of suite.tasks) { if (task.mode !== "run") continue; - if (isVitestTaskBenchmark(task)) { - await runAnalysisBench(task, suite, currentSuiteName, tinybenchModule); - } else if (task.type === "suite") { + if (task.type === "suite") { await runAnalysisBenchmarkSuite(task, tinybenchModule, currentSuiteName); + } else if (isVitestTaskBenchmark(task)) { + await runAnalysisBench(task, suite, currentSuiteName, tinybenchModule); } } diff --git a/packages/vitest-plugin/src/legacy/common.ts b/packages/vitest-plugin/src/legacy/common.ts index a76f9473..0791b963 100644 --- a/packages/vitest-plugin/src/legacy/common.ts +++ b/packages/vitest-plugin/src/legacy/common.ts @@ -1,7 +1,7 @@ import { getGitDir } from "@codspeed/core"; import path from "path"; -import { Benchmark, type RunnerTask, type RunnerTestSuite } from "vitest"; -import { getHooks } from "./compat"; +import { type RunnerTask, type RunnerTestSuite } from "vitest"; +import { getHooks, type BenchmarkTask } from "./compat"; type SuiteHooks = ReturnType; function getSuiteHooks(suite: RunnerTestSuite, name: keyof SuiteHooks) { @@ -36,6 +36,6 @@ export function patchRootSuiteWithFullFilePath(suite: RunnerTestSuite) { suite.name = path.relative(gitDir, suite.file.filepath); } -export function isVitestTaskBenchmark(task: RunnerTask): task is Benchmark { +export function isVitestTaskBenchmark(task: RunnerTask): task is BenchmarkTask { return task.type === "test" && task.meta.benchmark === true; } diff --git a/packages/vitest-plugin/src/legacy/compat.ts b/packages/vitest-plugin/src/legacy/compat.ts index d27baa71..a5802de7 100644 --- a/packages/vitest-plugin/src/legacy/compat.ts +++ b/packages/vitest-plugin/src/legacy/compat.ts @@ -1,13 +1,67 @@ -type VitestExports = typeof import("vitest"); +import type { RunnerTestCase, RunnerTestSuite } from "vitest"; +import type { Tinybench } from "../instrument"; + +/** + * A Vitest 3/4 benchmark task: a test case flagged as a benchmark, carrying the + * raw tinybench output on its result. + */ +export interface BenchmarkTask extends RunnerTestCase { + meta: RunnerTestCase["meta"] & { benchmark?: boolean }; +} + +/** The tinybench options Vitest 3/4 stores alongside a registered benchmark. */ +export interface LegacyBenchOptions { + time?: number; + warmupTime?: number; + warmupIterations?: number; + iterations?: number; + setup?: ( + task: { name: string }, + mode: "run" | "warmup", + ) => void | Promise; + teardown?: ( + task: { name: string }, + mode: "run" | "warmup", + ) => void | Promise; +} + +/** + * The Vitest 3/4 benchmark backend, which Vitest 5 removed: benchmarks now run + * through a `benchmark.provider` (see `v5/provider.ts`) and the runner is gone. + * The shapes are declared here because neither the removed `vitest/runners` and + * `vitest/suite` subpaths nor the Vitest 5 typings this package compiles against + * describe them. + */ +export interface LegacyBenchmarkApi { + NodeBenchmarkRunner: new (config?: unknown) => { + config: unknown; + runSuite(suite: RunnerTestSuite): Promise; + importTinybench(): Promise; + }; + getHooks: (suite: unknown) => Record unknown>>; + getBenchFn: (benchmark: BenchmarkTask) => () => unknown; + getBenchOptions: (benchmark: BenchmarkTask) => LegacyBenchOptions; +} /** * Vitest 4.1 moved the benchmark runner and the suite helpers to the main * `vitest` entry point and deprecated the `vitest/runners` and `vitest/suite` - * subpaths, which warn on import. + * subpaths, which warn on import. Both lookups have to be dynamic: which of the + * two shapes exists depends on the Vitest the user installed, and the subpaths + * don't resolve at all on Vitest 5. */ -async function resolveVitestApi() { - const { BenchmarkRunner, TestRunner }: Partial = - await import("vitest"); +async function resolveVitestApi(): Promise { + // Vitest 5 exports an unrelated `TestRunner`, so the namespace has to be + // re-typed from scratch rather than narrowed. + const { BenchmarkRunner, TestRunner } = + (await import("vitest")) as unknown as { + BenchmarkRunner?: LegacyBenchmarkApi["NodeBenchmarkRunner"]; + TestRunner?: { + getSuiteHooks: LegacyBenchmarkApi["getHooks"]; + getBenchFn: LegacyBenchmarkApi["getBenchFn"]; + getBenchOptions: LegacyBenchmarkApi["getBenchOptions"]; + }; + }; if (BenchmarkRunner && TestRunner) { return { @@ -18,8 +72,12 @@ async function resolveVitestApi() { }; } + // These subpaths only exist on Vitest 3/4, so they don't resolve when the + // plugin is installed alongside Vitest 5. const [runners, suite] = await Promise.all([ + // eslint-disable-next-line import/no-unresolved import("vitest/runners"), + // eslint-disable-next-line import/no-unresolved import("vitest/suite"), ]); diff --git a/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts b/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts index 274db403..be55606f 100644 --- a/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts +++ b/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts @@ -1,46 +1,19 @@ -// Vitest 3/4 exposed benchmark internals through `vitest/runners` and -// `vitest/suite`, and surfaced the `Benchmark` task type from `vitest`. Vitest 5 -// removed all of these (benchmarks now run through the unified `TestRunner`). +// Vitest 3/4 exposed the benchmark internals through the `vitest/runners` and +// `vitest/suite` subpaths, which Vitest 5 removed. `legacy/compat.ts` imports +// them only when the installed Vitest doesn't expose them from its main entry +// point, i.e. only on Vitest 3/4 — but the package is type-checked against +// whichever Vitest is installed, including 5, where the subpaths don't resolve. +// These declarations keep that code compiling; nothing imports them under v5. // -// The legacy runner (`analysis.ts`, `walltime/`) still imports them and is only -// loaded when the user runs Vitest 3/4, but the plugin is type-checked against -// whichever Vitest is installed — including 5, where these are gone. These -// ambient declarations keep the legacy code compiling there without affecting -// runtime: the modules are never imported under v5. - -import type * as tinybench from "tinybench"; - -declare module "vitest" { - import type { RunnerTestCase } from "vitest"; - - // In v3/4 a benchmark is a test case carrying tinybench output on its result. - interface Benchmark extends RunnerTestCase { - meta: RunnerTestCase["meta"] & { benchmark?: boolean }; - } -} +// This file must stay a script (no top-level imports): a `declare module` inside +// a module is an augmentation, which requires the module to resolve. declare module "vitest/runners" { - import type { RunnerTestSuite } from "vitest"; - - export class NodeBenchmarkRunner { - constructor(config?: unknown); - config: unknown; - runSuite(suite: RunnerTestSuite): Promise; - importTinybench(): Promise; - } + export const NodeBenchmarkRunner: import("./compat").LegacyBenchmarkApi["NodeBenchmarkRunner"]; } declare module "vitest/suite" { - import type { Benchmark } from "vitest"; - - export function getBenchFn(benchmark: Benchmark): () => unknown; - export function getBenchOptions(benchmark: Benchmark): { - time?: number; - warmupTime?: number; - warmupIterations?: number; - iterations?: number; - }; - export function getHooks( - suite: unknown, - ): Record unknown>>; + export const getHooks: import("./compat").LegacyBenchmarkApi["getHooks"]; + export const getBenchFn: import("./compat").LegacyBenchmarkApi["getBenchFn"]; + export const getBenchOptions: import("./compat").LegacyBenchmarkApi["getBenchOptions"]; } diff --git a/packages/vitest-plugin/src/legacy/walltime-utils.ts b/packages/vitest-plugin/src/legacy/walltime-utils.ts index 54bb4b1a..4e6cbf23 100644 --- a/packages/vitest-plugin/src/legacy/walltime-utils.ts +++ b/packages/vitest-plugin/src/legacy/walltime-utils.ts @@ -1,16 +1,12 @@ import { type Benchmark } from "@codspeed/core"; -import { - type RunnerTaskResult, - type RunnerTestSuite, - type Benchmark as VitestBenchmark, -} from "vitest"; -import { getBenchOptions } from "./compat"; +import { type RunnerTaskResult, type RunnerTestSuite } from "vitest"; import { tinybenchTaskToBenchmark, type TinybenchOptions, type TinybenchTask, } from "../instrument"; import { isVitestTaskBenchmark } from "./common"; +import { getBenchOptions, type BenchmarkTask } from "./compat"; export async function extractBenchmarkResults( suite: RunnerTestSuite, @@ -20,14 +16,14 @@ export async function extractBenchmarkResults( const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; for (const task of suite.tasks) { - if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { + if (task.type === "suite") { + const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); + benchmarks.push(...nestedBenchmarks); + } else if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { const benchmark = processBenchmarkTask(task, currentPath); if (benchmark) { benchmarks.push(benchmark); } - } else if (task.type === "suite") { - const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); - benchmarks.push(...nestedBenchmarks); } } @@ -35,7 +31,7 @@ export async function extractBenchmarkResults( } function processBenchmarkTask( - task: VitestBenchmark, + task: BenchmarkTask, suitePath: string, ): Benchmark | null { const uri = `${suitePath}::${task.name}`; diff --git a/packages/vitest-plugin/src/legacy/walltime.ts b/packages/vitest-plugin/src/legacy/walltime.ts index cd68eb84..535cb5ec 100644 --- a/packages/vitest-plugin/src/legacy/walltime.ts +++ b/packages/vitest-plugin/src/legacy/walltime.ts @@ -5,7 +5,6 @@ import { RunnerTaskResultPack, type RunnerTestSuite, } from "vitest"; -import { NodeBenchmarkRunner } from "./compat"; import { installInstrumentHooks, patchTaskRunOnce, @@ -13,6 +12,7 @@ import { type TinybenchBench, } from "../instrument"; import { patchRootSuiteWithFullFilePath } from "./common"; +import { NodeBenchmarkRunner } from "./compat"; import { extractBenchmarkResults } from "./walltime-utils"; type Tinybench = typeof tinybench; @@ -20,8 +20,8 @@ type Tinybench = typeof tinybench; /** * Lets tinybench run the benches through Vitest's default benchmark execution, * instrumenting each measured loop, then extracts the results from the suite - * tree afterwards. (The v5 seam instruments the same way but reads results off - * the live tinybench tasks instead — see `v5/setup.ts`.) + * tree afterwards. (The v5 provider instruments the same way but reads results + * off the live tinybench tasks instead — see `v5/provider.ts`.) */ export class WalltimeRunner extends NodeBenchmarkRunner { private suiteUris = new Map(); diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts new file mode 100644 index 00000000..9c5c31c0 --- /dev/null +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -0,0 +1,267 @@ +import { + getGitDir, + getInstrumentMode, + InstrumentHooks, + setupCore, + teardownCore, + type Benchmark, +} from "@codspeed/core"; +import { createRequire } from "module"; +import path from "path"; +import type { + BenchmarkGroup, + BenchmarkProvider, + BenchRegistrationInput, + BenchResult, + BenchRunOptions, +} from "vitest"; +import { + installInstrumentHooks, + rootFrameRegisterFn, + runAnalysisTask, + tinybenchTaskToBenchmark, + writeAndLogWalltimeResults, + type TinybenchBench, + type TinybenchFnOptions, + type TinybenchOptions, + type TinybenchTask, +} from "../instrument"; + +/** tinybench's statistics for one dimension (latency or throughput). */ +type BenchStatistics = BenchResult["latency"]; + +/** + * The subset of the host tinybench Bench the walltime path drives. The timing + * fields are the *resolved* options (tinybench exposes them as readonly + * instance properties), which is what the recorded benchmark config reports. + */ +interface TinybenchWithTasks extends TinybenchBench { + add: ( + name: string, + fn: BenchRegistrationInput["fn"], + fnOpts?: TinybenchFnOptions, + ) => unknown; + run: () => Promise; + tasks: TinybenchTask[]; + readonly iterations: number; + readonly time: number; + readonly warmup: boolean; + readonly warmupIterations: number; + readonly warmupTime: number; +} + +/** + * The host tinybench module, modeled structurally: the plugin's own tinybench + * dependency may be a different major, so its types can't describe the module + * resolved from the installed Vitest. `BenchRunOptions` is Vitest's re-export of + * the tinybench options it accepts, hence of the version it depends on. + */ +interface HostTinybench { + Bench: new (options?: BenchRunOptions) => TinybenchWithTasks; +} + +const isWalltime = getInstrumentMode() === "walltime"; + +// Vitest imports a provider module at most once per worker, and the plugin only +// wires this one up when CodSpeed drives the run (see `V5Backend`), so this is +// the per-worker setup. It must not be redone per benchmark: `setupCore()` +// truncates the process' perf map, which has to cover the whole worker. +setupCore(); +process.once("beforeExit", () => teardownCore()); + +/** + * Build the URI prefix shared by every benchmark of a group: the git-relative + * file path followed by the suite/test path, `::`-separated (e.g. + * `src/a.bench.ts::my suite::my test`). Each registration name is appended to + * it, so a group registering several benchmarks (`bench.compare()`) reports one + * URI per benchmark. + */ +function buildGroupUri(test: BenchmarkGroup["test"]): string { + const filepath = test.file?.filepath; + if (!filepath) { + throw new Error("[CodSpeed] could not resolve the running benchmark file"); + } + const gitDir = getGitDir(filepath); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + const relativeFile = path.relative(gitDir, filepath); + // `fullTestName` uses " > " between suite levels; normalize to "::". + const testPath = test.fullTestName.split(" > ").join("::"); + return [relativeFile, testPath].filter(Boolean).join("::"); +} + +/** + * Resolve the tinybench the *host* Vitest uses so walltime mode returns results + * in the exact shape Vitest serializes. The plugin's own tinybench may be a + * different major, so it is resolved relative to the installed Vitest. + */ +async function importHostTinybench(): Promise { + const require = createRequire(import.meta.url); + const vitestRequire = createRequire(require.resolve("vitest/package.json")); + return import(vitestRequire.resolve("tinybench")); +} + +function analysisResult(name: string): BenchResult { + // Zeroed statistics: enough to satisfy Vitest's reporter and its serializer. + const statistics: BenchStatistics = { + aad: 0, + critical: 0, + df: 0, + mad: 0, + max: 0, + mean: 0, + min: 0, + moe: 0, + p50: 0, + p75: 0, + p99: 0, + p995: 0, + p999: 0, + rme: 0, + samples: undefined, + samplesCount: 0, + sd: 0, + sem: 0, + variance: 0, + }; + + return { + name, + state: "completed", + latency: { ...statistics }, + throughput: { ...statistics }, + period: 0, + totalTime: 0, + runtime: "node", + runtimeVersion: process.versions.node, + timestampProviderName: "codspeed", + }; +} + +/** + * The CodSpeed benchmark provider. Two modes: + * - analysis (instrumentation/simulation): CodSpeed runs each fn itself under a + * tight instrument window and returns zeroed-but-valid results — the real + * measurement is captured by the instrument, not returned here. + * - walltime: tinybench drives the measured loop; CodSpeed brackets it with the + * instrument window and converts each task's stats into a result. + */ +const provider: BenchmarkProvider = { + async run({ test, registrations, options }): Promise { + // Resolve the URI up front, outside any measured window — it walks the + // filesystem (git root lookup), which must not land inside a sample. + const groupUri = buildGroupUri(test); + const getUri = (name: string) => `${groupUri}::${name}`; + + if (isWalltime) { + return runWalltime(registrations, options, getUri, test); + } + return runAnalysis(registrations, getUri); + }, +}; + +async function runAnalysis( + registrations: BenchRegistrationInput[], + getUri: (name: string) => string, +): Promise { + const label = InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; + const results: BenchResult[] = []; + for (const { name, fn, fnOpts } of registrations) { + const uri = getUri(name); + await runAnalysisTask({ fn, fnOpts }, uri); + console.log(`[CodSpeed] ${label} ${uri}`); + results.push(analysisResult(name)); + } + return results; +} + +async function runWalltime( + registrations: BenchRegistrationInput[], + options: BenchRunOptions | undefined, + getUri: (name: string) => string, + test: BenchmarkGroup["test"], +): Promise { + const { Bench } = await importHostTinybench(); + const bench = new Bench({ + signal: test.context.signal, + ...options, + // walltime needs per-iteration samples to compute quantiles + retainSamples: true, + }); + + for (const { name, fn, fnOpts } of registrations) { + // the root frame must be baked into the registered fn (tinybench v6 keeps + // `fn` in a private field, so it can't be wrapped after the fact) + bench.add(name, rootFrameRegisterFn(fn), fnOpts); + } + + installInstrumentHooks(bench, getUri); + const tasks = await bench.run(); + + throwOnErroredTasks(tasks); + collectWalltimeResults(bench, getUri); + + return tasks.map(tinybenchTaskToBenchResult); +} + +/** + * Surface benchmark failures the way Vitest's own provider does: a single error + * is rethrown as is, several are aggregated, and the test fails instead of + * reporting empty results. + */ +function throwOnErroredTasks(tasks: TinybenchTask[]): void { + const errors = tasks + .filter((task) => task.result?.state === "errored") + .map((task) => task.result?.error); + if (errors.length === 1) { + throw errors[0]; + } + if (errors.length > 1) { + throw new AggregateError(errors, "Some benchmarks failed"); + } +} + +function collectWalltimeResults( + bench: TinybenchWithTasks, + getUri: (name: string) => string, +): void { + const options: TinybenchOptions = { + time: bench.time, + iterations: bench.iterations, + // A bench with warmup turned off keeps its warmup timings, so they'd + // otherwise be reported as if a warmup had run. + warmupTime: bench.warmup ? bench.warmupTime : 0, + warmupIterations: bench.warmup ? bench.warmupIterations : 0, + }; + const benchmarks: Benchmark[] = []; + for (const task of bench.tasks) { + if (task.result?.state !== "completed") continue; + const benchmark = tinybenchTaskToBenchmark( + task, + getUri(task.name), + options, + ); + if (benchmark) { + benchmarks.push(benchmark); + } + } + writeAndLogWalltimeResults(benchmarks); +} + +function tinybenchTaskToBenchResult(task: TinybenchTask): BenchResult { + const result = task.result; + if (result?.state !== "completed") { + throw new Error( + `[CodSpeed] benchmark "${task.name}" did not complete: received "${result?.state ?? "no result"}"`, + ); + } + // tinybench's completed result already carries the full statistics surface + // Vitest serializes; pass it through, keyed by the task name. + return { + ...(result as unknown as Omit), + name: task.name, + }; +} + +export default provider; diff --git a/packages/vitest-plugin/src/v5/setup.ts b/packages/vitest-plugin/src/v5/setup.ts deleted file mode 100644 index 6a851463..00000000 --- a/packages/vitest-plugin/src/v5/setup.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { - getGitDir, - getInstrumentMode, - setupCore, - type Benchmark, -} from "@codspeed/core"; -import path from "path"; -// `TestRunner` is the unified runner Vitest 5 introduced; it replaces the -// `NodeBenchmarkRunner` the legacy seam subclasses, and is re-exported from the -// package root. It is read off the namespace (rather than a named import) so the -// plugin still type-checks against Vitest 3/4, which don't export it; this -// module only ever runs under v5. -import * as vitest from "vitest"; -import { - installInstrumentHooks, - patchTaskRunOnce, - tinybenchTaskToBenchmark, - writeAndLogWalltimeResults, - type TinybenchBench, - type TinybenchOptions, - type TinybenchTask, -} from "../instrument"; - -const TestRunner = (vitest as unknown as { TestRunner: unknown }).TestRunner; - -/** - * Vitest 5 runs benchmarks inside `test()` through `TestRunner`, calling the - * static `TestRunner.runBenchmarks(tinybench)` with a fully built tinybench - * instance. That static is referenced directly (not through `this`), so a - * runner subclass cannot intercept it — we patch the static in place instead. - * - * This module is registered as a Vitest `setupFile`, which runs in the worker - * before any test file is collected. Importing it installs the patch as a side - * effect. A setup file (rather than a custom `test.runner`) is used because it - * leaves the runner untouched for non-benchmark tests and also applies to the - * browser pool, where a Node runner file would not. - */ - -interface TinybenchWithTasks extends TinybenchBench { - name: string; - tasks: TinybenchTask[]; - run(): Promise; - // tinybench exposes the resolved options on the instance - opts?: TinybenchOptions; -} - -/** Minimal shape of the current Vitest test task we read for URI construction. */ -interface CurrentTest { - fullTestName?: string; - file?: { filepath: string }; -} - -function getCurrentTest(): CurrentTest | undefined { - // `getCurrentTest` is a static on the runner; typings don't surface it. - const getter = (TestRunner as unknown as { getCurrentTest?: () => unknown }) - .getCurrentTest; - return getter ? (getter() as CurrentTest | undefined) : undefined; -} - -/** - * Build the benchmark URI from the running test and the tinybench task name. - * Matches the legacy convention: git-relative file, then the suite/test path, - * then the bench name, all `::`-separated. - */ -function buildUri(taskName: string): string { - const test = getCurrentTest(); - const filepath = test?.file?.filepath; - if (!filepath) { - throw new Error("[CodSpeed] could not resolve the running benchmark file"); - } - const gitDir = getGitDir(filepath); - if (gitDir === undefined) { - throw new Error("Could not find a git repository"); - } - const relativeFile = path.relative(gitDir, filepath); - // `fullTestName` uses " > " between suite levels; normalize to "::". - const testPath = (test?.fullTestName ?? "").split(" > ").join("::"); - return [relativeFile, testPath, taskName].filter(Boolean).join("::"); -} - -function collectWalltimeResults(tinybench: TinybenchWithTasks): void { - const options: TinybenchOptions = tinybench.opts ?? {}; - const benchmarks: Benchmark[] = []; - - for (const task of tinybench.tasks) { - if (task.result?.state && task.result.state !== "completed") continue; - const benchmark = tinybenchTaskToBenchmark( - task, - buildUri(task.name), - options, - ); - if (benchmark) { - benchmarks.push(benchmark); - } - } - - writeAndLogWalltimeResults(benchmarks); -} - -function patchRunBenchmarks(): void { - const Runner = TestRunner as unknown as { - runBenchmarks: (tinybench: TinybenchWithTasks) => Promise; - }; - const originalRunBenchmarks = Runner.runBenchmarks.bind(TestRunner); - const isWalltime = getInstrumentMode() === "walltime"; - - Runner.runBenchmarks = async (tinybench: TinybenchWithTasks) => { - setupCore(); - - // tinybench's `Task` class isn't exported from the bench instance, so we - // reach it through a constructed task (Vitest adds them before running). - const TaskClass = tinybench.tasks[0]?.constructor as - | { prototype: { run: (this: unknown) => Promise } } - | undefined; - if (TaskClass) { - patchTaskRunOnce(TaskClass); - } - - installInstrumentHooks(tinybench, buildUri); - - const result = await originalRunBenchmarks(tinybench); - - if (isWalltime) { - collectWalltimeResults(tinybench); - } - - return result; - }; -} - -patchRunBenchmarks(); diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index db8c0d26..d9c707a1 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -25,30 +25,21 @@ export interface VitestBackend { /** * The `test` config fragment that wires the benchmark instrumentation into * Vitest: the V8 exec args (whose placement moved across versions) plus the - * integration seam (a custom runner on legacy, a setup file on v5). + * integration seam. Legacy wires a custom runner subclass (and, in walltime + * mode, asks tinybench to retain samples); v5 wires a `benchmark.provider` + * that owns execution and sample retention entirely. */ getBenchmarkTestConfig( v8Flags: string[], resolveFile: (name: string) => string, ): ViteUserConfig["test"]; - - /** - * The `test.benchmark` fragment asking tinybench to retain per-iteration - * samples so the walltime runner can compute quantiles. Only used in walltime - * mode. The option was renamed (`includeSamples` → `retainSamples`) when - * Vitest 5 moved to tinybench v6. - */ - getWalltimeBenchmarkConfig(): Record; } /** - * Vitest 5 reworked the benchmark backend: the dedicated `NodeBenchmarkRunner` - * and the `vitest/runners` / `vitest/suite` entrypoints are gone, benchmarks run - * inside `test()` through the unified `TestRunner`, and tinybench moved to v6. - * The integration seam therefore differs fundamentally (a `TestRunner` patch - * installed from a setup file vs. a runner subclass per mode), which is why the - * two backends are separate implementations rather than a pile of inline - * ternaries. + * The integration seam differs across Vitest generations: on 3/4 benchmarks run + * through a `NodeBenchmarkRunner` subclass (via `vitest/runners` / + * `vitest/suite`), while 5+ exposes a `benchmark.provider` API and dropped those + * entrypoints. * * When the version cannot be detected we assume the latest supported major. */ @@ -74,24 +65,26 @@ function getVitestMajorVersion(): number | null { } /** - * Vitest 5+. `vitest bench` runs under the `"test"` mode with - * `test.benchmark.enabled` flipped (there is no dedicated benchmark mode), so - * the plugin stays active for every mode and gates on the config instead. - * Instrumentation is installed from a setup file that patches the shared - * `TestRunner` (see `v5/setup.ts`). + * Vitest 5+. There is no dedicated benchmark Vite mode anymore (`vitest bench` + * runs under `"test"`), so the plugin stays active for every mode. + * Instrumentation is installed through a `benchmark.provider` that owns + * benchmark execution (see `v5/provider.ts`). */ class V5Backend implements VitestBackend { isActiveForViteMode(): boolean { return true; } - isBenchmarkRun(config: ViteUserConfig): boolean { - // `benchmark.enabled` only exists on the Vitest 5 config; the v3/4 typings - // we may be compiled against don't know about it. - const benchmark = config.test?.benchmark as - | { enabled?: boolean } - | undefined; - return benchmark?.enabled === true; + /** + * Vitest 5 clones its benchmark project from the resolved one *after* the Vite + * config hooks ran, so a benchmark run cannot be read off the incoming config: + * `test.benchmark.enabled` still holds the user's value and `vitest bench` + * only sets an internal CLI flag. CodSpeed exclusively drives benchmark runs, + * so gate on the instrument mode instead. Without CodSpeed the plugin injects + * nothing and Vitest runs the benchmarks through its own tinybench provider. + */ + isBenchmarkRun(): boolean { + return getInstrumentMode() !== "disabled"; } getBenchmarkTestConfig( @@ -100,13 +93,11 @@ class V5Backend implements VitestBackend { ): ViteUserConfig["test"] { return { execArgv: v8Flags, - setupFiles: [resolveFile("v5/setup")], + // The provider owns benchmark execution: it runs the registered functions + // under instrumentation (analysis) or drives tinybench itself (walltime). + benchmark: { provider: resolveFile("v5/provider") }, }; } - - getWalltimeBenchmarkConfig(): Record { - return { retainSamples: true }; - } } /** @@ -135,6 +126,12 @@ class LegacyBackend implements VitestBackend { ? undefined : resolveFile(join("legacy", instrumentMode)); + // Walltime asks tinybench to retain per-iteration samples so the runner can + // compute quantiles. On tinybench v2 (Vitest 3/4) the option is + // `includeSamples` (renamed `retainSamples` in v6). + const benchmark = + instrumentMode === "walltime" ? { includeSamples: true } : undefined; + return { // Vitest 3 nests exec args under `poolOptions.forks`; v4 moved them to a // top-level `test.execArgv`. @@ -143,10 +140,7 @@ class LegacyBackend implements VitestBackend { ? { execArgv: v8Flags } : { poolOptions: { forks: { execArgv: v8Flags } } }), ...(runner && { runner }), - }; - } - - getWalltimeBenchmarkConfig(): Record { - return { includeSamples: true }; + ...(benchmark && { benchmark }), + } as ViteUserConfig["test"]; } } diff --git a/packages/vitest-plugin/vitest.config.ts b/packages/vitest-plugin/vitest.config.ts index 444869bd..672e3a27 100644 --- a/packages/vitest-plugin/vitest.config.ts +++ b/packages/vitest-plugin/vitest.config.ts @@ -1,18 +1,6 @@ -import { createRequire } from "module"; import { defineConfig } from "vitest/config"; import codspeedPlugin from "./dist/index.mjs"; -// The legacy runner tests exercise the Vitest 3/4 benchmark backend -// (`vitest/suite`, `NodeBenchmarkRunner`), which Vitest 5 removed. Exclude them -// when running under v5+ so the file's static imports don't fail to resolve. -const require = createRequire(import.meta.url); -const vitestMajor = parseInt( - (require("vitest/package.json").version as string).split(".")[0], - 10, -); -const legacyOnlyTests = - vitestMajor >= 5 ? ["**/__tests__/instrumented.test.ts"] : []; - export default defineConfig({ // @ts-expect-error - TODO: investigate why importing from '.' wants to import only "main" field and thus fail plugins: [codspeedPlugin()], @@ -20,7 +8,7 @@ export default defineConfig({ __VERSION__: JSON.stringify("1.0.0"), }, test: { - exclude: ["**/tests/**/*", "**/.rollup.cache/**/*", ...legacyOnlyTests], + exclude: ["**/tests/**/*", "**/.rollup.cache/**/*"], mockReset: true, }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d6e150dd..040c2b91 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -255,6 +255,30 @@ importers: specifier: ^3.2.4 version: 3.2.4(@types/node@24.13.3)(lightningcss@1.33.0)(tsx@4.23.12)(yaml@2.9.0) + examples/with-vitest-v4: + devDependencies: + '@codspeed/vitest-plugin': + specifier: workspace:* + version: link:../../packages/vitest-plugin + typescript: + specifier: ^5.1.3 + version: 5.8.3 + vitest: + specifier: ^4.1.9 + version: 4.1.11(@types/node@24.13.3)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + + examples/with-vitest-v5: + devDependencies: + '@codspeed/vitest-plugin': + specifier: workspace:* + version: link:../../packages/vitest-plugin + typescript: + specifier: ^5.1.3 + version: 5.8.3 + vitest: + specifier: ^5.0.0 + version: 5.0.0(@types/node@24.13.3)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + packages/benchmark.js-plugin: dependencies: '@codspeed/core': @@ -368,8 +392,8 @@ importers: specifier: ^8.0.0 version: 8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) vitest: - specifier: ^4.1.11 - version: 4.1.11(@types/node@24.13.3)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + specifier: ^5.0.0 + version: 5.0.0(@types/node@24.13.3)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) packages: @@ -1831,8 +1855,8 @@ packages: '@jridgewell/trace-mapping@0.3.18': resolution: {integrity: sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==} - '@jridgewell/trace-mapping@0.3.29': - resolution: {integrity: sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==} + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} @@ -2762,6 +2786,17 @@ packages: vite: optional: true + '@vitest/mocker@5.0.0': + resolution: {integrity: sha512-66PGTMIiVJP3t4a5yxU9qPtf7MdTBs8jmToMvy+HVflB3Yy13WJZTtPePdvU+wjRV02SKK5doLbSA6o9pwOmiA==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + '@vitest/pretty-format@3.2.4': resolution: {integrity: sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==} @@ -2798,6 +2833,9 @@ packages: '@vitest/spy@4.1.11': resolution: {integrity: sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==} + '@vitest/spy@5.0.0': + resolution: {integrity: sha512-uy+luWBAPw9XfthoHi5AkfHUnuPYEESjl0p/r+meoBnU8bxg5GDQ3Ey8MjcJ6sqahkL4PFyrvfMJJBw7LbU06g==} + '@vitest/utils@3.2.4': resolution: {integrity: sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==} @@ -5119,6 +5157,9 @@ packages: resolution: {integrity: sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==} engines: {node: '>=12'} + magic-string@1.2.3: + resolution: {integrity: sha512-Bpb0W2TbLKOZ7vJnOUnVRGq3WL2p+ISV29M6hYPL1AFCpyKZpdr5ytiXoTSSxRVhg8YW7f65+6gbG8WG6PCa/g==} + make-dir@2.1.0: resolution: {integrity: sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==} engines: {node: '>=6'} @@ -5498,6 +5539,10 @@ packages: obug@2.1.1: resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + obug@2.1.4: + resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} + engines: {node: '>=12.20.0'} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -6387,6 +6432,10 @@ packages: resolution: {integrity: sha512-FlHoQpcFvCzeXK5kVPvV7IVgW/hs/B36QWTz876iSdeJguBDfdTSRQmYmaHX+fQNt4hp+gEFB2XXw+8hT4/y8A==} engines: {node: '>=20.0.0'} + tinybench@6.1.4: + resolution: {integrity: sha512-9APumHG7r4yOk4X4WlkmE71aZcv1gvin1czO3OQ1U9iJcFA5Ja/ygyb0vPOVHTthFozUYs8CLoLUlM8grb2lTQ==} + engines: {node: '>=20.0.0'} + tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} @@ -6394,6 +6443,10 @@ packages: resolution: {integrity: sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==} engines: {node: '>=18'} + tinyexec@1.3.0: + resolution: {integrity: sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==} + engines: {node: '>=18'} + tinyglobby@0.2.12: resolution: {integrity: sha512-qkf4trmKSIiMTs/E63cxH+ojC2unam7rJ0WrauAzpT3ECNTxGRMlaXxVbfxMUC/w0LaYk6jQ4y/nGR9uBO3tww==} engines: {node: '>=12.0.0'} @@ -6872,6 +6925,47 @@ packages: jsdom: optional: true + vitest@5.0.0: + resolution: {integrity: sha512-gpsMNoRhMjMktVxPtstOH4/PJuPyovVaMDr4oDilXaGH1EcqM2OE96SoHT2VIQ6fTGtTjqmHDrEu2X9RQiXf8Q==} + engines: {node: ^22.12.0 || ^24.0.0 || >=26.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 5.0.0 + '@vitest/browser-preview': 5.0.0 + '@vitest/browser-webdriverio': ^5.0.0-beta.5 || >=5.0.0 + '@vitest/coverage-istanbul': 5.0.0 + '@vitest/coverage-v8': 5.0.0 + '@vitest/ui': 5.0.0 + happy-dom: '*' + jsdom: '*' + vite: ^6.4.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + walk-up-path@3.0.1: resolution: {integrity: sha512-9YlCL/ynK3CTlrSRrDxZvUauLzAswPCrsaCgilqFevUYpeEW0/3ScEjaa3kbW/T0ghhkEr7mv+fpjqn1Y1YuTA==} @@ -7028,7 +7122,7 @@ snapshots: '@ampproject/remapping@2.3.0': dependencies: '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@apidevtools/json-schema-ref-parser@9.0.9': dependencies: @@ -7106,7 +7200,7 @@ snapshots: '@babel/parser': 7.28.0 '@babel/types': 7.28.1 '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 jsesc: 3.1.0 '@babel/helper-annotate-as-pure@7.22.5': @@ -8655,7 +8749,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@types/node': 24.13.3 chalk: 4.1.2 collect-v8-coverage: 1.0.2 @@ -8693,7 +8787,7 @@ snapshots: '@jest/source-map@29.6.3': dependencies: - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 callsites: 3.1.0 graceful-fs: 4.2.11 @@ -8749,7 +8843,7 @@ snapshots: dependencies: '@babel/core': 7.28.0 '@jest/types': 29.6.3 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 babel-plugin-istanbul: 6.1.1 chalk: 4.1.2 convert-source-map: 2.0.0 @@ -8786,7 +8880,7 @@ snapshots: '@jridgewell/gen-mapping@0.3.12': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@jridgewell/gen-mapping@0.3.3': dependencies: @@ -8815,7 +8909,7 @@ snapshots: '@jridgewell/resolve-uri': 3.1.0 '@jridgewell/sourcemap-codec': 1.4.14 - '@jridgewell/trace-mapping@0.3.29': + '@jridgewell/trace-mapping@0.3.31': dependencies: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 @@ -9800,6 +9894,15 @@ snapshots: optionalDependencies: vite: 8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) + '@vitest/mocker@5.0.0(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0))': + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + '@vitest/spy': 5.0.0 + estree-walker: 3.0.3 + magic-string: 1.2.3 + optionalDependencies: + vite: 8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) + '@vitest/pretty-format@3.2.4': dependencies: tinyrainbow: 2.0.0 @@ -9855,6 +9958,8 @@ snapshots: '@vitest/spy@4.1.11': {} + '@vitest/spy@5.0.0': {} + '@vitest/utils@3.2.4': dependencies: '@vitest/pretty-format': 3.2.4 @@ -12887,6 +12992,10 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.4.15 + magic-string@1.2.3: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + make-dir@2.1.0: dependencies: pify: 4.0.1 @@ -13354,6 +13463,8 @@ snapshots: obug@2.1.1: {} + obug@2.1.4: {} + once@1.4.0: dependencies: wrappy: 1.0.2 @@ -14304,10 +14415,14 @@ snapshots: tinybench@6.0.2: {} + tinybench@6.1.4: {} + tinyexec@0.3.2: {} tinyexec@1.0.2: {} + tinyexec@1.3.0: {} + tinyglobby@0.2.12: dependencies: fdir: 6.5.0(picomatch@4.0.7) @@ -14593,7 +14708,7 @@ snapshots: v8-to-istanbul@9.3.0: dependencies: - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@types/istanbul-lib-coverage': 2.0.6 convert-source-map: 2.0.0 @@ -14759,6 +14874,27 @@ snapshots: transitivePeerDependencies: - msw + vitest@5.0.0(@types/node@24.13.3)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)): + dependencies: + '@types/chai': 5.2.2 + '@vitest/mocker': 5.0.0(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + chai: 6.2.2 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 1.2.3 + obug: 2.1.4 + picomatch: 4.0.7 + std-env: 4.2.0 + tinybench: 6.1.4 + tinyexec: 1.3.0 + tinyglobby: 0.2.17 + vite: 8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 24.13.3 + transitivePeerDependencies: + - msw + walk-up-path@3.0.1: {} walker@1.0.8: From 2f7a55bbb40a2ea993339223543d05af3c859b03 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 4 Sep 2026 12:33:44 +0200 Subject: [PATCH 5/6] test(vitest-plugin): add with-vitest-v5 example Benchmark the plugin against Vitest 5 in CI the way the v3 and v4 examples do for their majors, using the `bench` fixture and `bench.compare()`. Refs COD-2931 Co-Authored-By: Claude --- examples/with-vitest-v5/package.json | 13 +++++++ .../with-vitest-v5/src/fibonacci.bench.ts | 38 +++++++++++++++++++ examples/with-vitest-v5/src/fibonacci.ts | 17 +++++++++ examples/with-vitest-v5/tsconfig.json | 13 +++++++ examples/with-vitest-v5/vitest.config.ts | 6 +++ 5 files changed, 87 insertions(+) create mode 100644 examples/with-vitest-v5/package.json create mode 100644 examples/with-vitest-v5/src/fibonacci.bench.ts create mode 100644 examples/with-vitest-v5/src/fibonacci.ts create mode 100644 examples/with-vitest-v5/tsconfig.json create mode 100644 examples/with-vitest-v5/vitest.config.ts diff --git a/examples/with-vitest-v5/package.json b/examples/with-vitest-v5/package.json new file mode 100644 index 00000000..440fd445 --- /dev/null +++ b/examples/with-vitest-v5/package.json @@ -0,0 +1,13 @@ +{ + "name": "with-vitest-v5", + "private": true, + "type": "module", + "scripts": { + "bench-vitest": "vitest bench --run" + }, + "devDependencies": { + "@codspeed/vitest-plugin": "workspace:*", + "typescript": "^5.1.3", + "vitest": "^5.0.0" + } +} diff --git a/examples/with-vitest-v5/src/fibonacci.bench.ts b/examples/with-vitest-v5/src/fibonacci.bench.ts new file mode 100644 index 00000000..3d303e3b --- /dev/null +++ b/examples/with-vitest-v5/src/fibonacci.bench.ts @@ -0,0 +1,38 @@ +import { describe, test } from "vitest"; +import { + iterativeFibonacci as iterativeFibonacciExport, + recursiveFibonacci as recursiveFibonacciExport, +} from "./fibonacci"; + +// Read the imported bindings once: Vite's module runner exposes exports through +// getters, and reading one inside the measured loop adds overhead to every +// iteration (Vitest warns about it). +const recursiveFibonacci = recursiveFibonacciExport; +const iterativeFibonacci = iterativeFibonacciExport; + +// Vitest 5 declares benchmarks through the `bench` test-context fixture: each +// `bench()` returns a registration that runs on `.run()`, or as part of a +// `bench.compare()` group. +describe("fibonacci", () => { + test("fibo 15", async ({ bench }) => { + await bench.compare( + bench("recursive", () => { + recursiveFibonacci(15); + }), + bench("iterative", () => { + iterativeFibonacci(15); + }), + ); + }); + + test("fibo 20", async ({ bench }) => { + await bench.compare( + bench("recursive", () => { + recursiveFibonacci(20); + }), + bench("iterative", () => { + iterativeFibonacci(20); + }), + ); + }); +}); diff --git a/examples/with-vitest-v5/src/fibonacci.ts b/examples/with-vitest-v5/src/fibonacci.ts new file mode 100644 index 00000000..94796660 --- /dev/null +++ b/examples/with-vitest-v5/src/fibonacci.ts @@ -0,0 +1,17 @@ +export function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +export function iterativeFibonacci(n: number): number { + let a = 0; + let b = 1; + for (let i = 0; i < n; i++) { + const temp = a + b; + a = b; + b = temp; + } + return a; +} diff --git a/examples/with-vitest-v5/tsconfig.json b/examples/with-vitest-v5/tsconfig.json new file mode 100644 index 00000000..ace1de03 --- /dev/null +++ b/examples/with-vitest-v5/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "verbatimModuleSyntax": true, + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/examples/with-vitest-v5/vitest.config.ts b/examples/with-vitest-v5/vitest.config.ts new file mode 100644 index 00000000..4b1290c1 --- /dev/null +++ b/examples/with-vitest-v5/vitest.config.ts @@ -0,0 +1,6 @@ +import codspeedPlugin from "@codspeed/vitest-plugin"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [codspeedPlugin()], +}); From eb2042ea711614ff503854928c8476869a10aff5 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 4 Sep 2026 12:33:46 +0200 Subject: [PATCH 6/6] docs(vitest-plugin): show the vitest 5 bench fixture API in the readme Benchmarks are declared through the `bench` test-context fixture on Vitest 5; keep the top-level `bench()` form as a note for Vitest 3 and 4. Drop the fallback log line from the sample output, which the plugin no longer prints. Refs COD-2931 Co-Authored-By: Claude --- packages/vitest-plugin/README.md | 34 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/packages/vitest-plugin/README.md b/packages/vitest-plugin/README.md index 6cba8821..47021a0f 100644 --- a/packages/vitest-plugin/README.md +++ b/packages/vitest-plugin/README.md @@ -41,10 +41,10 @@ pnpm add --save-dev @codspeed/vitest-plugin vitest ## Usage -Let's create a fibonacci function and benchmark it with `vitest.bench`: +Let's create a fibonacci function and benchmark it with Vitest's `bench` fixture: ```ts title="benches/fibo.bench.ts" -import { describe, bench } from "vitest"; +import { describe, test } from "vitest"; function fibonacci(n: number): number { if (n < 2) { @@ -54,16 +54,33 @@ function fibonacci(n: number): number { } describe("fibonacci", () => { - bench("fibonacci10", () => { - fibonacci(10); - }); - - bench("fibonacci15", () => { - fibonacci(15); + test("depth", async ({ bench }) => { + await bench.compare( + bench("fibonacci10", () => { + fibonacci(10); + }), + bench("fibonacci15", () => { + fibonacci(15); + }), + ); }); }); ``` +> [!NOTE] +> The `bench` fixture is a Vitest 5 API. On Vitest 3 and 4, benchmarks are +> declared with the top-level `bench()` export instead: +> +> ```ts +> import { bench, describe } from "vitest"; +> +> describe("fibonacci", () => { +> bench("fibonacci10", () => { +> fibonacci(10); +> }); +> }); +> ``` + Create or update your `vitest.config.ts` file to use the CodSpeed runner: ```ts title="vitest.config.ts" @@ -80,7 +97,6 @@ Finally, run your benchmarks (here with `pnpm`): ```bash $ pnpm vitest bench --run -[CodSpeed] bench detected but no instrumentation found, falling back to default vitest runner ... Regular `vitest bench` output ```