diff --git a/.gitignore b/.gitignore index 5233f61e..3cfb1201 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,5 @@ packages/app/.env # turbo .turbo/ .rollup.cache/ + +.codspeed diff --git a/CLAUDE.md b/CLAUDE.md index b82671a6..00a76a53 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,4 +94,25 @@ Based on the codebase analysis, to add stats access features: - Build: `pnpm turbo run build --filter=` - Typecheck: `pnpm turbo run typecheck --filter=` - Lint: `pnpm turbo run lint --filter=` - - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`). \ No newline at end of file + - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`). + +## Testing a plugin during development + +A plugin behaves differently depending on whether CodSpeed is driving the run, +so exercise all three of these when developing or reviewing a plugin change +(build the plugin first — the benches import from `dist`): + +1. **Fallback (not under CodSpeed).** No env vars. The plugin must stay out of + the way and let the framework run its benchmarks normally (no instrumentation, + no hijacked output). e.g. `pnpm turbo run bench --filter=`. +2. **Instrumentation / simulation.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=simulation` + (or `instrumentation`). The plugin hijacks the run to do a single instrumented + pass per benchmark and prints `Measured/Checked ` instead of the normal + harness output. +3. **Walltime.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=walltime`. The plugin + instruments the framework's real benchmark loop and collects walltime results. + +Running these locally outside the CodSpeed runner is expected to log +`instrument-hooks: failed to write environment.json` and skip actual measurement +writes — the point is to verify the plugin's control flow and output per mode, +not to produce real measurements. \ No newline at end of file diff --git a/examples/with-vitest-v4/package.json b/examples/with-vitest-v4/package.json new file mode 100644 index 00000000..8a67c417 --- /dev/null +++ b/examples/with-vitest-v4/package.json @@ -0,0 +1,13 @@ +{ + "name": "with-vitest-v4", + "private": true, + "type": "module", + "scripts": { + "bench-vitest": "vitest bench --run" + }, + "devDependencies": { + "@codspeed/vitest-plugin": "workspace:*", + "typescript": "^5.1.3", + "vitest": "^4.1.9" + } +} diff --git a/examples/with-vitest-v4/src/fibonacci.bench.ts b/examples/with-vitest-v4/src/fibonacci.bench.ts new file mode 100644 index 00000000..227a6e67 --- /dev/null +++ b/examples/with-vitest-v4/src/fibonacci.bench.ts @@ -0,0 +1,20 @@ +import { bench, describe } from "vitest"; +import { iterativeFibonacci, recursiveFibonacci } from "./fibonacci"; + +describe("fibonacci", () => { + bench("recursive fibo 15", () => { + recursiveFibonacci(15); + }); + + bench("recursive fibo 20", () => { + recursiveFibonacci(20); + }); + + bench("iterative fibo 15", () => { + iterativeFibonacci(15); + }); + + bench("iterative fibo 20", () => { + iterativeFibonacci(20); + }); +}); diff --git a/examples/with-vitest-v4/src/fibonacci.ts b/examples/with-vitest-v4/src/fibonacci.ts new file mode 100644 index 00000000..94796660 --- /dev/null +++ b/examples/with-vitest-v4/src/fibonacci.ts @@ -0,0 +1,17 @@ +export function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +export function iterativeFibonacci(n: number): number { + let a = 0; + let b = 1; + for (let i = 0; i < n; i++) { + const temp = a + b; + a = b; + b = temp; + } + return a; +} diff --git a/examples/with-vitest-v4/tsconfig.json b/examples/with-vitest-v4/tsconfig.json new file mode 100644 index 00000000..ace1de03 --- /dev/null +++ b/examples/with-vitest-v4/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "verbatimModuleSyntax": true, + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/examples/with-vitest-v4/vitest.config.ts b/examples/with-vitest-v4/vitest.config.ts new file mode 100644 index 00000000..4b1290c1 --- /dev/null +++ b/examples/with-vitest-v4/vitest.config.ts @@ -0,0 +1,6 @@ +import codspeedPlugin from "@codspeed/vitest-plugin"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [codspeedPlugin()], +}); diff --git a/examples/with-vitest-v5/package.json b/examples/with-vitest-v5/package.json new file mode 100644 index 00000000..440fd445 --- /dev/null +++ b/examples/with-vitest-v5/package.json @@ -0,0 +1,13 @@ +{ + "name": "with-vitest-v5", + "private": true, + "type": "module", + "scripts": { + "bench-vitest": "vitest bench --run" + }, + "devDependencies": { + "@codspeed/vitest-plugin": "workspace:*", + "typescript": "^5.1.3", + "vitest": "^5.0.0" + } +} diff --git a/examples/with-vitest-v5/src/fibonacci.bench.ts b/examples/with-vitest-v5/src/fibonacci.bench.ts new file mode 100644 index 00000000..3d303e3b --- /dev/null +++ b/examples/with-vitest-v5/src/fibonacci.bench.ts @@ -0,0 +1,38 @@ +import { describe, test } from "vitest"; +import { + iterativeFibonacci as iterativeFibonacciExport, + recursiveFibonacci as recursiveFibonacciExport, +} from "./fibonacci"; + +// Read the imported bindings once: Vite's module runner exposes exports through +// getters, and reading one inside the measured loop adds overhead to every +// iteration (Vitest warns about it). +const recursiveFibonacci = recursiveFibonacciExport; +const iterativeFibonacci = iterativeFibonacciExport; + +// Vitest 5 declares benchmarks through the `bench` test-context fixture: each +// `bench()` returns a registration that runs on `.run()`, or as part of a +// `bench.compare()` group. +describe("fibonacci", () => { + test("fibo 15", async ({ bench }) => { + await bench.compare( + bench("recursive", () => { + recursiveFibonacci(15); + }), + bench("iterative", () => { + iterativeFibonacci(15); + }), + ); + }); + + test("fibo 20", async ({ bench }) => { + await bench.compare( + bench("recursive", () => { + recursiveFibonacci(20); + }), + bench("iterative", () => { + iterativeFibonacci(20); + }), + ); + }); +}); diff --git a/examples/with-vitest-v5/src/fibonacci.ts b/examples/with-vitest-v5/src/fibonacci.ts new file mode 100644 index 00000000..94796660 --- /dev/null +++ b/examples/with-vitest-v5/src/fibonacci.ts @@ -0,0 +1,17 @@ +export function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +export function iterativeFibonacci(n: number): number { + let a = 0; + let b = 1; + for (let i = 0; i < n; i++) { + const temp = a + b; + a = b; + b = temp; + } + return a; +} diff --git a/examples/with-vitest-v5/tsconfig.json b/examples/with-vitest-v5/tsconfig.json new file mode 100644 index 00000000..ace1de03 --- /dev/null +++ b/examples/with-vitest-v5/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "verbatimModuleSyntax": true, + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/examples/with-vitest-v5/vitest.config.ts b/examples/with-vitest-v5/vitest.config.ts new file mode 100644 index 00000000..4b1290c1 --- /dev/null +++ b/examples/with-vitest-v5/vitest.config.ts @@ -0,0 +1,6 @@ +import codspeedPlugin from "@codspeed/vitest-plugin"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [codspeedPlugin()], +}); diff --git a/packages/vitest-plugin/README.md b/packages/vitest-plugin/README.md index 6cba8821..47021a0f 100644 --- a/packages/vitest-plugin/README.md +++ b/packages/vitest-plugin/README.md @@ -41,10 +41,10 @@ pnpm add --save-dev @codspeed/vitest-plugin vitest ## Usage -Let's create a fibonacci function and benchmark it with `vitest.bench`: +Let's create a fibonacci function and benchmark it with Vitest's `bench` fixture: ```ts title="benches/fibo.bench.ts" -import { describe, bench } from "vitest"; +import { describe, test } from "vitest"; function fibonacci(n: number): number { if (n < 2) { @@ -54,16 +54,33 @@ function fibonacci(n: number): number { } describe("fibonacci", () => { - bench("fibonacci10", () => { - fibonacci(10); - }); - - bench("fibonacci15", () => { - fibonacci(15); + test("depth", async ({ bench }) => { + await bench.compare( + bench("fibonacci10", () => { + fibonacci(10); + }), + bench("fibonacci15", () => { + fibonacci(15); + }), + ); }); }); ``` +> [!NOTE] +> The `bench` fixture is a Vitest 5 API. On Vitest 3 and 4, benchmarks are +> declared with the top-level `bench()` export instead: +> +> ```ts +> import { bench, describe } from "vitest"; +> +> describe("fibonacci", () => { +> bench("fibonacci10", () => { +> fibonacci(10); +> }); +> }); +> ``` + Create or update your `vitest.config.ts` file to use the CodSpeed runner: ```ts title="vitest.config.ts" @@ -80,7 +97,6 @@ Finally, run your benchmarks (here with `pnpm`): ```bash $ pnpm vitest bench --run -[CodSpeed] bench detected but no instrumentation found, falling back to default vitest runner ... Regular `vitest bench` output ``` diff --git a/packages/vitest-plugin/benches/flat.bench.ts b/packages/vitest-plugin/benches/flat.bench.ts index 74e67b9d..b9b494a4 100644 --- a/packages/vitest-plugin/benches/flat.bench.ts +++ b/packages/vitest-plugin/benches/flat.bench.ts @@ -1,5 +1,10 @@ -import { bench, describe } from "vitest"; -import parsePr from "./parsePr"; +import { describe, test } from "vitest"; +import parsePrExport from "./parsePr"; + +// Read the imported binding once: Vite's module runner exposes exports through +// getters, and reading one inside the measured loop adds overhead to every +// iteration (Vitest warns about it). +const parsePr = parsePrExport; const LONG_BODY = new Array(1_000) @@ -9,12 +14,15 @@ const LONG_BODY = .join("\n") + "fixes #123"; describe("parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test-1", number: 1 }); - }); - - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test-2", number: 2 }); + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", () => { + parsePr({ body: "fixes #123", title: "test-1", number: 1 }); + }), + bench("long body", () => { + parsePr({ body: LONG_BODY, title: "test-2", number: 2 }); + }), + ); }); }); @@ -24,10 +32,13 @@ function fibo(n: number): number { } describe("fibo", () => { - bench("fibo 10", () => { - fibo(10); - }); - bench("fibo 15", () => { - fibo(15); + test("depth", async ({ bench }) => { + await bench("fibo 10", () => { + fibo(10); + }).run(); + + await bench("fibo 15", () => { + fibo(15); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/hooks.bench.ts b/packages/vitest-plugin/benches/hooks.bench.ts index d790ca3c..c706dda8 100644 --- a/packages/vitest-plugin/benches/hooks.bench.ts +++ b/packages/vitest-plugin/benches/hooks.bench.ts @@ -1,37 +1,39 @@ -import { - afterAll, - afterEach, - beforeAll, - beforeEach, - bench, - describe, - expect, -} from "vitest"; +import { describe, expect, test } from "vitest"; +// Exercises tinybench's per-benchmark hooks, which Vitest 5 exposes through the +// `bench(name, options, fn)` options object (`beforeAll`/`beforeEach`/...). describe("hooks", () => { let count = 0; - describe("run", () => { - beforeAll(() => { + + const hooks = { + beforeAll: () => { count += 10; - }); - beforeEach(() => { + }, + beforeEach: () => { count += 1; - }); - afterEach(() => { + }, + afterEach: () => { count -= 1; - }); - afterAll(() => { + }, + afterAll: () => { count -= 10; - }); + }, + }; - bench("one", () => { - expect(count).toBe(11); - }); - bench("two", () => { - expect(count).toBe(11); - }); + test("hooked benches", async ({ bench }) => { + await bench.compare( + bench("one", hooks, () => { + expect(count).toBe(11); + }), + bench("two", hooks, () => { + expect(count).toBe(11); + }), + ); }); - bench("end", () => { - expect(count).toBe(0); + + test("after the hooked benches", async ({ bench }) => { + await bench("count is back to zero", () => { + expect(count).toBe(0); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/macos.bench.ts b/packages/vitest-plugin/benches/macos.bench.ts index e05a3082..792e9c3f 100644 --- a/packages/vitest-plugin/benches/macos.bench.ts +++ b/packages/vitest-plugin/benches/macos.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe } from "vitest"; +import { describe, test } from "vitest"; const isMacOS = process.platform === "darwin"; @@ -10,7 +10,9 @@ function fibo(n: number): number { // macOS-only benchmark: skipped on every other platform, so it only runs on // the `codspeed-walltime-macos` CI job (see .github/workflows/codspeed.yml). describe.skipIf(!isMacOS)("macos only", () => { - bench("fibo darwin", () => { - fibo(30); + test("fibo", async ({ bench }) => { + await bench("fibo darwin", () => { + fibo(30); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/parsePr.bench.ts b/packages/vitest-plugin/benches/parsePr.bench.ts index 91c0adee..60829edf 100644 --- a/packages/vitest-plugin/benches/parsePr.bench.ts +++ b/packages/vitest-plugin/benches/parsePr.bench.ts @@ -1,5 +1,10 @@ -import { bench, describe } from "vitest"; -import parsePr from "./parsePr"; +import { describe, test } from "vitest"; +import parsePrExport from "./parsePr"; + +// Read the imported binding once: Vite's module runner exposes exports through +// getters, and reading one inside the measured loop adds overhead to every +// iteration (Vitest warns about it). +const parsePr = parsePrExport; const LONG_BODY = new Array(1_000) @@ -8,44 +13,49 @@ const LONG_BODY = ) .join("\n") + "fixes #123"; -describe("parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); - }); +function benchShortBody() { + parsePr({ body: "fixes #123", title: "test", number: 124 }); +} - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); +function benchLongBody() { + parsePr({ body: LONG_BODY, title: "test", number: 124 }); +} + +describe("parsePr", () => { + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", benchShortBody), + bench("long body", benchLongBody), + ); }); describe("nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); - }); - - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", benchShortBody), + bench("long body", benchLongBody), + ); }); describe("deeply nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); }); }); }); describe("another parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); - }); - - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", benchShortBody), + bench("long body", benchLongBody), + ); }); describe("nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); }); }); diff --git a/packages/vitest-plugin/benches/timing.bench.ts b/packages/vitest-plugin/benches/timing.bench.ts index 83331ad6..fd6aa5f8 100644 --- a/packages/vitest-plugin/benches/timing.bench.ts +++ b/packages/vitest-plugin/benches/timing.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe, type BenchOptions } from "vitest"; +import { describe, test, type BenchCompareOptions } from "vitest"; const busySleep = (ms: number): void => { const end = performance.now() + ms; @@ -7,33 +7,24 @@ const busySleep = (ms: number): void => { } }; -const timingBenchOptions: BenchOptions = { +const timingBenchOptions: BenchCompareOptions = { iterations: 5, warmupIterations: 0, }; describe("timing tests", () => { - bench( - "wait 1ms", - async () => { - busySleep(1); - }, - timingBenchOptions, - ); - - bench( - "wait 500ms", - async () => { - busySleep(500); - }, - timingBenchOptions, - ); - - bench( - "wait 1sec", - async () => { - busySleep(1_000); - }, - timingBenchOptions, - ); + test("busy sleep", async ({ bench }) => { + await bench.compare( + bench("wait 1ms", async () => { + busySleep(1); + }), + bench("wait 500ms", async () => { + busySleep(500); + }), + bench("wait 1sec", async () => { + busySleep(1_000); + }), + timingBenchOptions, + ); + }); }); diff --git a/packages/vitest-plugin/package.json b/packages/vitest-plugin/package.json index 3c935d88..ab23e739 100644 --- a/packages/vitest-plugin/package.json +++ b/packages/vitest-plugin/package.json @@ -41,13 +41,13 @@ "peerDependencies": { "tinybench": ">=2.9.0", "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0", - "vitest": "^3.2 || ^4" + "vitest": "^3.2 || ^4 || ^5" }, "devDependencies": { "@total-typescript/shoehorn": "^0.1.1", "execa": "^8.0.1", "tinybench": "^2.9.0", "vite": "^8.0.0", - "vitest": "^4.1.11" + "vitest": "^5.0.0" } } diff --git a/packages/vitest-plugin/rollup.config.mjs b/packages/vitest-plugin/rollup.config.mjs index 2fda2b6a..0c1a0948 100644 --- a/packages/vitest-plugin/rollup.config.mjs +++ b/packages/vitest-plugin/rollup.config.mjs @@ -23,18 +23,26 @@ export default defineConfig([ plugins: jsPlugins(pkg.version), external: ["@codspeed/core", /^vitest/], }, + // The built layout mirrors the source layout (dist/legacy/*, dist/v5/*) so the + // plugin resolves the seam files with one path rule in both dev and prod. { - input: "src/analysis.ts", - output: { file: "dist/analysis.mjs", format: "es" }, + input: "src/legacy/analysis.ts", + output: { file: "dist/legacy/analysis.mjs", format: "es" }, // top-level await plugins: jsPlugins(pkg.version, "es2022"), external: ["@codspeed/core", /^vitest/], }, { - input: "src/walltime/index.ts", - output: { file: "dist/walltime.mjs", format: "es" }, + input: "src/legacy/walltime.ts", + output: { file: "dist/legacy/walltime.mjs", format: "es" }, // top-level await plugins: jsPlugins(pkg.version, "es2022"), external: ["@codspeed/core", /^vitest/], }, + { + input: "src/v5/provider.ts", + output: { file: "dist/v5/provider.mjs", format: "es" }, + plugins: jsPlugins(pkg.version), + external: ["@codspeed/core", /^vitest/, "tinybench"], + }, ]); diff --git a/packages/vitest-plugin/src/__tests__/globalSetup.test.ts b/packages/vitest-plugin/src/__tests__/globalSetup.test.ts index 178b3ecb..f475f048 100644 --- a/packages/vitest-plugin/src/__tests__/globalSetup.test.ts +++ b/packages/vitest-plugin/src/__tests__/globalSetup.test.ts @@ -4,19 +4,27 @@ import globalSetup from "../globalSetup"; console.log = vi.fn(); describe("globalSetup", () => { - it("should log the correct message on setup and teardown, and fail when teardown is called twice", async () => { + it("should log setup and teardown once, even when Vitest runs them per project", async () => { const teardown = globalSetup(); expect(console.log).toHaveBeenCalledWith( "[CodSpeed] @codspeed/vitest-plugin v1.0.0 - setup", ); + // Vitest 5 runs the same globalSetup for the base project and for the + // benchmark project it clones from it. + globalSetup(); + + expect(console.log).toHaveBeenCalledTimes(1); + teardown(); expect(console.log).toHaveBeenCalledWith( "[CodSpeed] @codspeed/vitest-plugin v1.0.0 - teardown", ); - expect(() => teardown()).toThrowError("teardown called twice"); + teardown(); + + expect(console.log).toHaveBeenCalledTimes(2); }); }); diff --git a/packages/vitest-plugin/src/__tests__/index.test.ts b/packages/vitest-plugin/src/__tests__/index.test.ts index 63ce93cc..da284190 100644 --- a/packages/vitest-plugin/src/__tests__/index.test.ts +++ b/packages/vitest-plugin/src/__tests__/index.test.ts @@ -1,5 +1,5 @@ -import { fromPartial } from "@total-typescript/shoehorn"; import { getV8Flags } from "@codspeed/core"; +import { fromPartial } from "@total-typescript/shoehorn"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import codspeedPlugin from "../index"; @@ -55,6 +55,7 @@ describe("codSpeedPlugin", () => { // Clean up environment variables delete process.env.CODSPEED_ENV; delete process.env.CODSPEED_RUNNER_MODE; + fsMocks.setMockVersion("4.0.18"); }); it("should have a name", async () => { @@ -66,7 +67,9 @@ describe("codSpeedPlugin", () => { }); describe("apply", () => { - it("should not apply the plugin when the mode is not benchmark", async () => { + it("should not apply the plugin when the mode is not benchmark (v3/v4)", async () => { + fsMocks.setMockVersion("4.0.18"); + const applyPlugin = applyPluginFunction( {}, fromPartial({ mode: "test" }), @@ -75,7 +78,8 @@ describe("codSpeedPlugin", () => { expect(applyPlugin).toBe(false); }); - it("should apply the plugin when there is no instrumentation", async () => { + it("should apply the plugin when there is no instrumentation (v3/v4)", async () => { + fsMocks.setMockVersion("4.0.18"); coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(false); const applyPlugin = applyPluginFunction( @@ -89,7 +93,8 @@ describe("codSpeedPlugin", () => { expect(applyPlugin).toBe(true); }); - it("should apply the plugin when there is instrumentation", async () => { + it("should apply the plugin when there is instrumentation (v3/v4)", async () => { + fsMocks.setMockVersion("4.0.18"); coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true); const applyPlugin = applyPluginFunction( @@ -99,14 +104,32 @@ describe("codSpeedPlugin", () => { expect(applyPlugin).toBe(true); }); + + it("should stay active regardless of mode on v5 (benchmark gating happens in config)", async () => { + fsMocks.setMockVersion("5.0.0"); + coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true); + + const applyPlugin = applyPluginFunction( + {}, + fromPartial({ mode: "test" }), + ); + + expect(applyPlugin).toBe(true); + fsMocks.setMockVersion("4.0.18"); + }); }); it("should apply the codspeed config for v4", () => { + fsMocks.setMockVersion("4.0.18"); const config = resolvedCodSpeedPlugin.config; if (typeof config !== "function") throw new Error("config is not a function"); - const result = config.call({} as never, {}, fromPartial({})); + const result = config.call( + {} as never, + {}, + fromPartial({ mode: "benchmark" }), + ); expect(result).toStrictEqual({ test: { @@ -116,23 +139,25 @@ describe("codSpeedPlugin", () => { pool: "forks", execArgv: getV8Flags(), runner: expect.stringContaining( - "packages/vitest-plugin/src/analysis.ts", + "packages/vitest-plugin/src/legacy/analysis.ts", ), }, }); }); it("should apply the codspeed config for v3 with poolOptions", () => { - // Set mock version to v3 fsMocks.setMockVersion("3.2.0"); - // Create a new plugin instance to pick up the mocked version const v3Plugin = codspeedPlugin(); const config = v3Plugin.config; if (typeof config !== "function") throw new Error("config is not a function"); - const result = config.call({} as never, {}, fromPartial({})); + const result = config.call( + {} as never, + {}, + fromPartial({ mode: "benchmark" }), + ); expect(result).toStrictEqual({ test: { @@ -146,12 +171,65 @@ describe("codSpeedPlugin", () => { }, }, runner: expect.stringContaining( - "packages/vitest-plugin/src/analysis.ts", + "packages/vitest-plugin/src/legacy/analysis.ts", ), }, }); - // Reset mock version back to v4 fsMocks.setMockVersion("4.0.18"); }); + + describe("v5 config", () => { + it("should not inject config when CodSpeed is not driving the run", () => { + fsMocks.setMockVersion("5.0.0"); + delete process.env.CODSPEED_ENV; + + const v5Plugin = codspeedPlugin(); + const config = v5Plugin.config; + if (typeof config !== "function") + throw new Error("config is not a function"); + + const result = config.call( + {} as never, + {}, + fromPartial({ mode: "test" }), + ); + + expect(result).toBeUndefined(); + process.env.CODSPEED_ENV = "1"; + fsMocks.setMockVersion("4.0.18"); + }); + + it("should wire the v5 benchmark provider (not a runner or setup file)", () => { + fsMocks.setMockVersion("5.0.0"); + const v5Plugin = codspeedPlugin(); + const config = v5Plugin.config; + if (typeof config !== "function") + throw new Error("config is not a function"); + + const result = config.call( + {} as never, + {}, + fromPartial({ mode: "test" }), + ); + + expect(result).toStrictEqual({ + test: { + globalSetup: [ + expect.stringContaining( + "packages/vitest-plugin/src/globalSetup.ts", + ), + ], + pool: "forks", + execArgv: getV8Flags(), + benchmark: { + provider: expect.stringContaining( + "packages/vitest-plugin/src/v5/provider.ts", + ), + }, + }, + }); + fsMocks.setMockVersion("4.0.18"); + }); + }); }); diff --git a/packages/vitest-plugin/src/__tests__/instrumented.test.ts b/packages/vitest-plugin/src/__tests__/instrumented.test.ts index 443d62f2..c3e9178a 100644 --- a/packages/vitest-plugin/src/__tests__/instrumented.test.ts +++ b/packages/vitest-plugin/src/__tests__/instrumented.test.ts @@ -1,7 +1,11 @@ import { fromPartial } from "@total-typescript/shoehorn"; import { describe, expect, it, vi, type RunnerTestSuite } from "vitest"; -import { AnalysisRunner as CodSpeedRunner } from "../analysis"; -import { getBenchFn, getBenchOptions } from "../compat"; +import { AnalysisRunner as CodSpeedRunner } from "../legacy/analysis"; +import { getBenchFn, getBenchOptions } from "../legacy/compat"; + +// `legacy/compat` resolves the Vitest 3/4 benchmark backend from the installed +// Vitest, so it is mocked entirely: that backend is gone on Vitest 5, which this +// package is developed against. const coreMocks = vi.hoisted(() => { return { @@ -28,10 +32,16 @@ vi.mock("@codspeed/core", async (importOriginal) => { console.log = vi.fn(); -vi.mock("../compat", async (importOriginal) => { - const actual = await importOriginal(); +vi.mock("../legacy/compat", () => { + class NodeBenchmarkRunner { + async importTinybench() { + return import("tinybench"); + } + } + return { - ...actual, + NodeBenchmarkRunner, + getHooks: vi.fn(), getBenchFn: vi.fn(), getBenchOptions: vi.fn(), }; diff --git a/packages/vitest-plugin/src/compat.ts b/packages/vitest-plugin/src/compat.ts deleted file mode 100644 index d27baa71..00000000 --- a/packages/vitest-plugin/src/compat.ts +++ /dev/null @@ -1,35 +0,0 @@ -type VitestExports = typeof import("vitest"); - -/** - * Vitest 4.1 moved the benchmark runner and the suite helpers to the main - * `vitest` entry point and deprecated the `vitest/runners` and `vitest/suite` - * subpaths, which warn on import. - */ -async function resolveVitestApi() { - const { BenchmarkRunner, TestRunner }: Partial = - await import("vitest"); - - if (BenchmarkRunner && TestRunner) { - return { - NodeBenchmarkRunner: BenchmarkRunner, - getHooks: TestRunner.getSuiteHooks, - getBenchFn: TestRunner.getBenchFn, - getBenchOptions: TestRunner.getBenchOptions, - }; - } - - const [runners, suite] = await Promise.all([ - import("vitest/runners"), - import("vitest/suite"), - ]); - - return { - NodeBenchmarkRunner: runners.NodeBenchmarkRunner, - getHooks: suite.getHooks, - getBenchFn: suite.getBenchFn, - getBenchOptions: suite.getBenchOptions, - }; -} - -export const { NodeBenchmarkRunner, getHooks, getBenchFn, getBenchOptions } = - await resolveVitestApi(); diff --git a/packages/vitest-plugin/src/globalSetup.ts b/packages/vitest-plugin/src/globalSetup.ts index 8c5e1a34..143dafea 100644 --- a/packages/vitest-plugin/src/globalSetup.ts +++ b/packages/vitest-plugin/src/globalSetup.ts @@ -9,13 +9,21 @@ function logCodSpeed(message: string) { console.log(`[CodSpeed] ${message}`); } +let setupHappened = false; let teardownHappened = false; +// Vitest 5 clones a dedicated `(bench)` project from the base one and runs the +// same globalSetup module for both, so setup and teardown each fire twice. +// Report only the first pass and make the repeats no-ops; throwing on the second +// teardown fails the whole run during close. export default function () { - logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`); + if (!setupHappened) { + setupHappened = true; + logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`); + } return () => { - if (teardownHappened) throw new Error("teardown called twice"); + if (teardownHappened) return; teardownHappened = true; logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - teardown`); diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index 2bdf59ed..cbbe5be0 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -1,5 +1,4 @@ import { - getCodspeedRunnerMode, getInstrumentMode, getV8Flags, InstrumentHooks, @@ -7,49 +6,33 @@ import { SetupInstrumentsRequestBody, SetupInstrumentsResponse, } from "@codspeed/core"; -import { readFileSync } from "fs"; -import { createRequire } from "module"; import { join } from "path"; import { Plugin } from "vite"; import { type ViteUserConfig } from "vitest/config"; +import { resolveVitestBackend } from "./vitestBackend"; // get this file's directory path from import.meta.url const __dirname = new URL(".", import.meta.url).pathname; const isFileInTs = import.meta.url.endsWith(".ts"); -function getCodSpeedFileFromName(name: string) { +/** + * Resolve a plugin-owned file (globalSetup, seam entry points) shipped alongside + * this module. Source (`.ts`) and built (`.mjs`) layouts are kept identical (see + * rollup.config.ts), so the same relative `name` works in both. + */ +function resolveFile(name: string): string { const fileExtension = isFileInTs ? "ts" : "mjs"; - return join(__dirname, `${name}.${fileExtension}`); } -function getVitestMajorVersion(): number | null { - try { - // Resolve vitest from the project's perspective (cwd), not from the plugin's location - // This ensures we detect the vitest version the user has installed - const require = createRequire(join(process.cwd(), "package.json")); - const vitestPkgPath = require.resolve("vitest/package.json"); - const vitestPkg = JSON.parse(readFileSync(vitestPkgPath, "utf-8")); - return parseInt(vitestPkg.version.split(".")[0], 10); - } catch { - return null; - } -} - -function getRunnerFile(): string | undefined { - const instrumentMode = getInstrumentMode(); - if (instrumentMode === "disabled") { - return undefined; - } - - return getCodSpeedFileFromName(instrumentMode); -} - export default function codspeedPlugin(): Plugin { + // Resolved lazily on each hook rather than once here: the installed Vitest + // version is detected from the project's cwd, which isn't reliably knowable at + // plugin-construction time (and tests swap it between construction and use). return { name: "codspeed:vitest", apply(_, { mode }) { - if (mode !== "benchmark") { + if (!resolveVitestBackend().isActiveForViteMode(mode)) { return false; } if ( @@ -61,38 +44,17 @@ export default function codspeedPlugin(): Plugin { return true; }, enforce: "post", - config(): ViteUserConfig { - const runnerFile = getRunnerFile(); - const runnerMode = getCodspeedRunnerMode(); - const v8Flags = getV8Flags(); - const vitestMajorVersion = getVitestMajorVersion(); - // by default, assume Vitest v4 or higher - const isVitestV4OrHigher = (vitestMajorVersion ?? 4) >= 4; + config(incomingConfig, { mode }): ViteUserConfig | undefined { + const backend = resolveVitestBackend(); + if (!backend.isBenchmarkRun(incomingConfig, mode)) { + return undefined; + } const config: ViteUserConfig = { test: { pool: "forks", - ...(isVitestV4OrHigher - ? { execArgv: v8Flags } - : { - // Compat with Vitest v3 - // See: https://vitest.dev/guide/migration.html#pool-rework - // poolOptions only exists in Vitest v3 - poolOptions: { - forks: { - execArgv: v8Flags, - }, - }, - }), - globalSetup: [getCodSpeedFileFromName("globalSetup")], - ...(runnerFile && { - runner: runnerFile, - }), - ...(runnerMode === "walltime" && { - benchmark: { - includeSamples: true, - }, - }), + globalSetup: [resolveFile("globalSetup")], + ...backend.getBenchmarkTestConfig(getV8Flags(), resolveFile), }, }; diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts new file mode 100644 index 00000000..cd069946 --- /dev/null +++ b/packages/vitest-plugin/src/instrument.ts @@ -0,0 +1,364 @@ +import { + calculateQuantiles, + InstrumentHooks, + MARKER_TYPE_BENCHMARK_END, + MARKER_TYPE_BENCHMARK_START, + msToNs, + msToS, + optimizeFunction, + optimizeFunctionSync, + wrapWithRootFrame, + wrapWithRootFrameSync, + writeWalltimeResults, + type Benchmark, + type BenchmarkStats, +} from "@codspeed/core"; +import type * as tinybench from "tinybench"; + +export type Tinybench = typeof tinybench; + +/** tinybench's per-task lifecycle hooks (a subset of `FnOptions`). */ +export interface TinybenchFnOptions { + beforeAll?: (mode?: "run" | "warmup") => unknown; + beforeEach?: (mode?: "run" | "warmup") => unknown; + afterEach?: (mode?: "run" | "warmup") => unknown; + afterAll?: (mode?: "run" | "warmup") => unknown; +} + +/** The captured registration for a task: its fn and options. */ +export interface CapturedTask { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fn: (...args: any[]) => any; + fnOpts?: TinybenchFnOptions; +} + +/** A tinybench task, exposing the `fn` the runner wraps with the root frame. */ +export interface TinybenchTask { + name: string; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fn: (...args: any[]) => any; + result?: TinybenchTaskResult; +} + +/** tinybench's per-task setup/teardown hook signature. */ +export type TinybenchHook = ( + task: TinybenchTask, + mode: "run" | "warmup", +) => Promise | void; + +/** The mutable subset of a tinybench Bench the runner reaches into. */ +export interface TinybenchBench { + setup: TinybenchHook; + teardown: TinybenchHook; +} + +/** + * The tinybench statistics shape (latency/throughput) shared across the v2 and + * v6 lines. Only the fields the conversion needs are modeled. + */ +interface TinybenchStatistics { + min: number; + max: number; + mean: number; + sd: number; + samples: number[] | undefined; +} + +interface TinybenchTaskResult { + state?: string; + /** Set by tinybench when `state` is `"errored"`. */ + error?: Error; + totalTime: number; + latency: TinybenchStatistics; +} + +/** The subset of tinybench bench options that maps onto a CodSpeed benchmark config. */ +export interface TinybenchOptions { + time?: number; + warmupTime?: number; + warmupIterations?: number; + iterations?: number; +} + +/** Timestamp marking the open edge of a task's measured loop. */ +interface InstrumentWindow { + runStart: bigint | null; +} + +/** + * The window bracketing the currently running task's measured loop, driven by + * the setup/teardown hooks below. Tasks run strictly sequentially within a + * worker, so one shared value suffices. + */ +const instrumentWindow: InstrumentWindow = { runStart: null }; + +/** The tinybench Task prototype whose `run` the legacy seam wraps. */ +interface TinybenchTaskClass { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prototype: { run: (this: any) => Promise }; +} + +let isTaskPatched = false; + +/** + * Wrap every task's fn with the root frame by patching `Task.prototype.run` in + * place. Used only by the legacy (Vitest 3/4) walltime seam, which runs on + * tinybench v2 where a task's `fn` is a plain, reassignable property. + * + * The Vitest 5 seam cannot use this: tinybench v6 made `fn` a true `#private` + * field, so reassigning `task.fn` is a silent no-op there — the frame must be + * baked in at registration time instead (see rootFrameRegisterFn). + */ +export function patchTaskRunOnce(TaskClass: TinybenchTaskClass): void { + if (isTaskPatched) { + return; + } + isTaskPatched = true; + + const originalRun = TaskClass.prototype.run; + TaskClass.prototype.run = async function (this: CapturedTask) { + const originalFn = this.fn; + this.fn = wrapWithRootFrame(() => originalFn.call(this)); + + try { + return await originalRun.call(this); + } finally { + this.fn = originalFn; + } + }; +} + +/** + * The root-frame wrap to hand tinybench at registration time. Post-hoc + * assignment to a task's `fn` is a no-op on tinybench v6 (private field), so the + * frame must be baked into the registered fn instead. + */ +export function rootFrameRegisterFn( + fn: CapturedTask["fn"], +): CapturedTask["fn"] { + return wrapWithRootFrame(() => fn()); +} + +/** + * Run one benchmark under instrumentation, matching the analysis window the + * Vitest 3/4 runner uses exactly: warm the JIT with `optimizeFunction` outside + * the window, run the user hooks around a single measured `fn()`, and bracket + * only that call with `startBenchmark`/`stopBenchmark` under the root frame. The + * measurement comes from the instrument, so no wall-clock markers are emitted + * and tinybench's timing loop is not involved. + * + * Synchronous benchmarks run through a fully synchronous window + * (`wrapWithRootFrameSync`, no `await`): awaiting a sync fn would splice Node's + * promise-hook machinery in above the root frame and pollute the sample. Async + * benchmarks necessarily use the awaited path. + */ +export async function runAnalysisTask( + { fn, fnOpts }: CapturedTask, + uri: string, +): Promise { + if (isAsyncFn(fn)) { + await runAnalysisTaskAsync(fn, fnOpts, uri); + } else { + await runAnalysisTaskSync(fn, fnOpts, uri); + } +} + +function isAsyncFn(fn: CapturedTask["fn"]): boolean { + return fn.constructor?.name === "AsyncFunction"; +} + +async function runAnalysisTaskAsync( + fn: CapturedTask["fn"], + fnOpts: TinybenchFnOptions | undefined, + uri: string, +): Promise { + await fnOpts?.beforeAll?.("run"); + await optimizeFunction(async () => { + await fnOpts?.beforeEach?.("run"); + await fn(); + await fnOpts?.afterEach?.("run"); + }); + + await fnOpts?.beforeEach?.("run"); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await fnOpts?.afterEach?.("run"); + await fnOpts?.afterAll?.("run"); +} + +function runAnalysisTaskSync( + fn: CapturedTask["fn"], + fnOpts: TinybenchFnOptions | undefined, + uri: string, +): void { + fnOpts?.beforeAll?.("run"); + optimizeFunctionSync(() => { + fnOpts?.beforeEach?.("run"); + fn(); + fnOpts?.afterEach?.("run"); + }); + + fnOpts?.beforeEach?.("run"); + global.gc?.(); + wrapWithRootFrameSync(() => { + InstrumentHooks.startBenchmark(); + fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + fnOpts?.afterEach?.("run"); + fnOpts?.afterAll?.("run"); +} + +/** + * Drive the instrumentation window from each bench's run-mode setup/teardown + * hooks so it brackets only tinybench's measured loop, excluding the warmup + * that runs beforehand and the statistics computation tinybench performs after + * the loop. Wrapping the whole `Task.run()` would otherwise fold all of that + * framework overhead into the recorded sample. + * + * User-provided hooks are preserved and keep their order relative to the work + * under test. + */ +export function installInstrumentHooks( + bench: TinybenchBench, + getUri: (taskName: string) => string, +): void { + const userSetup = bench.setup; + const userTeardown = bench.teardown; + + bench.setup = async (task, mode) => { + await userSetup(task, mode); + if (mode === "run") { + InstrumentHooks.startBenchmark(); + instrumentWindow.runStart = InstrumentHooks.currentTimestamp(); + } + }; + + bench.teardown = async (task, mode) => { + if (mode === "run") { + closeInstrumentWindow(getUri(task.name)); + } + await userTeardown(task, mode); + }; +} + +function closeInstrumentWindow(uri: string): void { + emitBenchmarkWindow(uri, instrumentWindow.runStart!); + instrumentWindow.runStart = null; +} + +/** + * Close the currently open instrumentation window: emit the benchmark markers + * bracketing [start, now], stop the benchmark, and attribute the sample to `uri`. + * + * Benchmark markers must land inside the sample window opened by + * startBenchmark(), so they are emitted before stopBenchmark() closes it. The + * runner consumes the FIFO stream in order, so a marker sent after stopBenchmark + * would fall outside the sample and break the expected + * SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. + */ +function emitBenchmarkWindow(uri: string, start: bigint): void { + const end = InstrumentHooks.currentTimestamp(); + const pid = process.pid; + + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, start); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, end); + + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(pid, uri); +} + +/** + * Persist collected walltime benchmarks, if any. The per-seam result traversal + * differs (legacy walks the Vitest suite tree, v5 iterates the live tinybench + * tasks) because each generation exposes results differently, but the write and + * the summary log are identical. + */ +export function writeAndLogWalltimeResults(benchmarks: Benchmark[]): void { + if (benchmarks.length === 0) { + return; + } + writeWalltimeResults(benchmarks); + console.log( + `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.`, + ); +} + +/** + * Convert a completed tinybench task into a CodSpeed walltime benchmark. Returns + * null when the task produced no samples (e.g. fully optimized out), in which + * case there is nothing to record. + */ +export function tinybenchTaskToBenchmark( + task: TinybenchTask, + uri: string, + options: TinybenchOptions, +): Benchmark | null { + const stats = tinybenchResultToStats(task.result, options); + if (stats === null) { + return null; + } + + return { + name: task.name, + uri, + config: { + max_rounds: options.iterations ?? null, + max_time_ns: options.time ? msToNs(options.time) : null, + min_round_time_ns: null, // tinybench does not have an option for this + warmup_time_ns: + options.warmupIterations !== 0 && options.warmupTime + ? msToNs(options.warmupTime) + : null, + }, + stats, + }; +} + +function tinybenchResultToStats( + result: TinybenchTaskResult | undefined, + options: TinybenchOptions, +): BenchmarkStats | null { + if (!result) { + throw new Error("No benchmark data available in result"); + } + + const { totalTime, latency } = result; + const { min, max, mean, sd, samples } = latency; + + const sortedTimesNs = (samples ?? []).map(msToNs).sort((a, b) => a - b); + const meanNs = msToNs(mean); + const stdevNs = msToNs(sd); + + if (sortedTimesNs.length == 0) { + // Sometimes the benchmarks can be completely optimized out and not even + // run, but their beforeEach and afterEach hooks are still executed, and the + // task is still considered a success. + return null; + } + + const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = + calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); + + return { + min_ns: msToNs(min), + max_ns: msToNs(max), + mean_ns: meanNs, + stdev_ns: stdevNs, + q1_ns, + median_ns, + q3_ns, + total_time: msToS(totalTime), + iter_per_round: 1, // tinybench runs one iteration per round + rounds: sortedTimesNs.length, + iqr_outlier_rounds, + stdev_outlier_rounds, + warmup_iters: options.warmupIterations ?? 0, + }; +} diff --git a/packages/vitest-plugin/src/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts similarity index 84% rename from packages/vitest-plugin/src/analysis.ts rename to packages/vitest-plugin/src/legacy/analysis.ts index dc50a5e2..336e2392 100644 --- a/packages/vitest-plugin/src/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -8,13 +8,19 @@ import { wrapWithRootFrame, } from "@codspeed/core"; import type * as tinybench from "tinybench"; -import { Benchmark, type RunnerTestSuite } from "vitest"; +import { type RunnerTestSuite } from "vitest"; + import { callSuiteHook, isVitestTaskBenchmark, patchRootSuiteWithFullFilePath, } from "./common"; -import { getBenchFn, getBenchOptions, NodeBenchmarkRunner } from "./compat"; +import { + getBenchFn, + getBenchOptions, + NodeBenchmarkRunner, + type BenchmarkTask, +} from "./compat"; type Tinybench = typeof tinybench; @@ -33,13 +39,17 @@ function logCodSpeed(message: string) { } async function runAnalysisBench( - benchmark: Benchmark, + benchmark: BenchmarkTask, suite: RunnerTestSuite, currentSuiteName: string, tinybenchModule: Tinybench, ) { const uri = `${currentSuiteName}::${benchmark.name}`; - const fn = getBenchFn(benchmark); + // tinybench's bench fn carries a `this: Bench` requirement on Vitest 3/4 that + // we don't need (the work under test is self-contained); call it as a plain + // parameterless function. The cast also smooths over the typing differences + // across supported Vitest versions. + const fn = getBenchFn(benchmark) as () => unknown; // Constructing a Bench applies tinybench's no-op defaults for the setup and // teardown hooks and gives them the Task they expect. The bench itself is @@ -50,7 +60,6 @@ async function runAnalysisBench( await bench.setup(task, "warmup"); await optimizeFunction(async () => { await callSuiteHook(suite, benchmark, "beforeEach"); - // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench await fn(); await callSuiteHook(suite, benchmark, "afterEach"); }); @@ -62,7 +71,6 @@ async function runAnalysisBench( global.gc?.(); await wrapWithRootFrame(async () => { InstrumentHooks.startBenchmark(); - // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench await fn(); InstrumentHooks.stopBenchmark(); InstrumentHooks.setExecutedBenchmark(process.pid, uri); @@ -88,10 +96,10 @@ async function runAnalysisBenchmarkSuite( for (const task of suite.tasks) { if (task.mode !== "run") continue; - if (isVitestTaskBenchmark(task)) { - await runAnalysisBench(task, suite, currentSuiteName, tinybenchModule); - } else if (task.type === "suite") { + if (task.type === "suite") { await runAnalysisBenchmarkSuite(task, tinybenchModule, currentSuiteName); + } else if (isVitestTaskBenchmark(task)) { + await runAnalysisBench(task, suite, currentSuiteName, tinybenchModule); } } diff --git a/packages/vitest-plugin/src/common.ts b/packages/vitest-plugin/src/legacy/common.ts similarity index 74% rename from packages/vitest-plugin/src/common.ts rename to packages/vitest-plugin/src/legacy/common.ts index e853c307..0791b963 100644 --- a/packages/vitest-plugin/src/common.ts +++ b/packages/vitest-plugin/src/legacy/common.ts @@ -1,7 +1,7 @@ import { getGitDir } from "@codspeed/core"; import path from "path"; -import { Benchmark, type RunnerTask, type RunnerTestSuite } from "vitest"; -import { getHooks } from "./compat"; +import { type RunnerTask, type RunnerTestSuite } from "vitest"; +import { getHooks, type BenchmarkTask } from "./compat"; type SuiteHooks = ReturnType; function getSuiteHooks(suite: RunnerTestSuite, name: keyof SuiteHooks) { @@ -19,8 +19,9 @@ export async function callSuiteHook( const hooks = getSuiteHooks(suite, name); - // @ts-expect-error TODO: add support for hooks parameters - await Promise.all(hooks.map((fn) => fn())); + // TODO: add support for hook parameters. The hook signature differs across + // supported Vitest versions, so we call them through a parameterless cast. + await Promise.all((hooks as Array<() => unknown>).map((fn) => fn())); if (name === "afterEach" && suite?.suite) { await callSuiteHook(suite.suite, currentTask, name); @@ -35,6 +36,6 @@ export function patchRootSuiteWithFullFilePath(suite: RunnerTestSuite) { suite.name = path.relative(gitDir, suite.file.filepath); } -export function isVitestTaskBenchmark(task: RunnerTask): task is Benchmark { +export function isVitestTaskBenchmark(task: RunnerTask): task is BenchmarkTask { return task.type === "test" && task.meta.benchmark === true; } diff --git a/packages/vitest-plugin/src/legacy/compat.ts b/packages/vitest-plugin/src/legacy/compat.ts new file mode 100644 index 00000000..a5802de7 --- /dev/null +++ b/packages/vitest-plugin/src/legacy/compat.ts @@ -0,0 +1,93 @@ +import type { RunnerTestCase, RunnerTestSuite } from "vitest"; +import type { Tinybench } from "../instrument"; + +/** + * A Vitest 3/4 benchmark task: a test case flagged as a benchmark, carrying the + * raw tinybench output on its result. + */ +export interface BenchmarkTask extends RunnerTestCase { + meta: RunnerTestCase["meta"] & { benchmark?: boolean }; +} + +/** The tinybench options Vitest 3/4 stores alongside a registered benchmark. */ +export interface LegacyBenchOptions { + time?: number; + warmupTime?: number; + warmupIterations?: number; + iterations?: number; + setup?: ( + task: { name: string }, + mode: "run" | "warmup", + ) => void | Promise; + teardown?: ( + task: { name: string }, + mode: "run" | "warmup", + ) => void | Promise; +} + +/** + * The Vitest 3/4 benchmark backend, which Vitest 5 removed: benchmarks now run + * through a `benchmark.provider` (see `v5/provider.ts`) and the runner is gone. + * The shapes are declared here because neither the removed `vitest/runners` and + * `vitest/suite` subpaths nor the Vitest 5 typings this package compiles against + * describe them. + */ +export interface LegacyBenchmarkApi { + NodeBenchmarkRunner: new (config?: unknown) => { + config: unknown; + runSuite(suite: RunnerTestSuite): Promise; + importTinybench(): Promise; + }; + getHooks: (suite: unknown) => Record unknown>>; + getBenchFn: (benchmark: BenchmarkTask) => () => unknown; + getBenchOptions: (benchmark: BenchmarkTask) => LegacyBenchOptions; +} + +/** + * Vitest 4.1 moved the benchmark runner and the suite helpers to the main + * `vitest` entry point and deprecated the `vitest/runners` and `vitest/suite` + * subpaths, which warn on import. Both lookups have to be dynamic: which of the + * two shapes exists depends on the Vitest the user installed, and the subpaths + * don't resolve at all on Vitest 5. + */ +async function resolveVitestApi(): Promise { + // Vitest 5 exports an unrelated `TestRunner`, so the namespace has to be + // re-typed from scratch rather than narrowed. + const { BenchmarkRunner, TestRunner } = + (await import("vitest")) as unknown as { + BenchmarkRunner?: LegacyBenchmarkApi["NodeBenchmarkRunner"]; + TestRunner?: { + getSuiteHooks: LegacyBenchmarkApi["getHooks"]; + getBenchFn: LegacyBenchmarkApi["getBenchFn"]; + getBenchOptions: LegacyBenchmarkApi["getBenchOptions"]; + }; + }; + + if (BenchmarkRunner && TestRunner) { + return { + NodeBenchmarkRunner: BenchmarkRunner, + getHooks: TestRunner.getSuiteHooks, + getBenchFn: TestRunner.getBenchFn, + getBenchOptions: TestRunner.getBenchOptions, + }; + } + + // These subpaths only exist on Vitest 3/4, so they don't resolve when the + // plugin is installed alongside Vitest 5. + const [runners, suite] = await Promise.all([ + // eslint-disable-next-line import/no-unresolved + import("vitest/runners"), + // eslint-disable-next-line import/no-unresolved + import("vitest/suite"), + ]); + + return { + NodeBenchmarkRunner: runners.NodeBenchmarkRunner, + getHooks: suite.getHooks, + getBenchFn: suite.getBenchFn, + getBenchOptions: suite.getBenchOptions, + }; +} + +export const { NodeBenchmarkRunner, getHooks, getBenchFn, getBenchOptions } = + await resolveVitestApi(); diff --git a/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts b/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts new file mode 100644 index 00000000..be55606f --- /dev/null +++ b/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts @@ -0,0 +1,19 @@ +// Vitest 3/4 exposed the benchmark internals through the `vitest/runners` and +// `vitest/suite` subpaths, which Vitest 5 removed. `legacy/compat.ts` imports +// them only when the installed Vitest doesn't expose them from its main entry +// point, i.e. only on Vitest 3/4 — but the package is type-checked against +// whichever Vitest is installed, including 5, where the subpaths don't resolve. +// These declarations keep that code compiling; nothing imports them under v5. +// +// This file must stay a script (no top-level imports): a `declare module` inside +// a module is an augmentation, which requires the module to resolve. + +declare module "vitest/runners" { + export const NodeBenchmarkRunner: import("./compat").LegacyBenchmarkApi["NodeBenchmarkRunner"]; +} + +declare module "vitest/suite" { + export const getHooks: import("./compat").LegacyBenchmarkApi["getHooks"]; + export const getBenchFn: import("./compat").LegacyBenchmarkApi["getBenchFn"]; + export const getBenchOptions: import("./compat").LegacyBenchmarkApi["getBenchOptions"]; +} diff --git a/packages/vitest-plugin/src/legacy/walltime-utils.ts b/packages/vitest-plugin/src/legacy/walltime-utils.ts new file mode 100644 index 00000000..4e6cbf23 --- /dev/null +++ b/packages/vitest-plugin/src/legacy/walltime-utils.ts @@ -0,0 +1,106 @@ +import { type Benchmark } from "@codspeed/core"; +import { type RunnerTaskResult, type RunnerTestSuite } from "vitest"; +import { + tinybenchTaskToBenchmark, + type TinybenchOptions, + type TinybenchTask, +} from "../instrument"; +import { isVitestTaskBenchmark } from "./common"; +import { getBenchOptions, type BenchmarkTask } from "./compat"; + +export async function extractBenchmarkResults( + suite: RunnerTestSuite, + parentPath = "", +): Promise { + const benchmarks: Benchmark[] = []; + const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; + + for (const task of suite.tasks) { + if (task.type === "suite") { + const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); + benchmarks.push(...nestedBenchmarks); + } else if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { + const benchmark = processBenchmarkTask(task, currentPath); + if (benchmark) { + benchmarks.push(benchmark); + } + } + } + + return benchmarks; +} + +function processBenchmarkTask( + task: BenchmarkTask, + suitePath: string, +): Benchmark | null { + const uri = `${suitePath}::${task.name}`; + + const result = task.result; + if (!result) { + console.warn(` ⚠ No result data available for ${uri}`); + return null; + } + + try { + const benchOptions = getBenchOptions(task); + const benchmark = tinybenchTaskToBenchmark( + adaptLegacyResult(task.name, result), + uri, + benchOptions as TinybenchOptions, + ); + + if (benchmark === null) { + console.log(` ✔ No walltime data to collect for ${uri}`); + return null; + } + + console.log(` ✔ Collected walltime data for ${uri}`); + return benchmark; + } catch (error) { + console.warn(` ⚠ Failed to process benchmark result for ${uri}:`, error); + return null; + } +} + +/** + * Vitest 3/4 attaches the raw tinybench v2 result under `result.benchmark`, + * whose statistics are a flat object ({ totalTime, min, max, mean, sd, samples }). + * Reshape it into the `{ result: { totalTime, latency } }` form the shared + * converter expects (tinybench v6 nests statistics under `latency`). + */ +interface LegacyBenchmarkStats { + totalTime: number; + min: number; + max: number; + mean: number; + sd: number; + samples: number[]; +} + +function adaptLegacyResult( + name: string, + result: RunnerTaskResult, +): TinybenchTask { + // `result.benchmark` only exists on the Vitest 3/4 task result; the v5 typings + // (compiled against here) dropped it. + const benchmark = (result as { benchmark?: LegacyBenchmarkStats }).benchmark; + if (!benchmark) { + throw new Error("No benchmark data available in result"); + } + + return { + name, + fn: () => undefined, + result: { + totalTime: benchmark.totalTime, + latency: { + min: benchmark.min, + max: benchmark.max, + mean: benchmark.mean, + sd: benchmark.sd, + samples: benchmark.samples, + }, + }, + }; +} diff --git a/packages/vitest-plugin/src/legacy/walltime.ts b/packages/vitest-plugin/src/legacy/walltime.ts new file mode 100644 index 00000000..535cb5ec --- /dev/null +++ b/packages/vitest-plugin/src/legacy/walltime.ts @@ -0,0 +1,119 @@ +import { setupCore } from "@codspeed/core"; +import type * as tinybench from "tinybench"; +import { + RunnerTaskEventPack, + RunnerTaskResultPack, + type RunnerTestSuite, +} from "vitest"; +import { + installInstrumentHooks, + patchTaskRunOnce, + writeAndLogWalltimeResults, + type TinybenchBench, +} from "../instrument"; +import { patchRootSuiteWithFullFilePath } from "./common"; +import { NodeBenchmarkRunner } from "./compat"; +import { extractBenchmarkResults } from "./walltime-utils"; + +type Tinybench = typeof tinybench; + +/** + * Lets tinybench run the benches through Vitest's default benchmark execution, + * instrumenting each measured loop, then extracts the results from the suite + * tree afterwards. (The v5 provider instruments the same way but reads results + * off the live tinybench tasks instead — see `v5/provider.ts`.) + */ +export class WalltimeRunner extends NodeBenchmarkRunner { + private suiteUris = new Map(); + /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks + private currentSuiteId: string | null = null; + + async runSuite(suite: RunnerTestSuite): Promise { + patchRootSuiteWithFullFilePath(suite); + this.populateBenchmarkUris(suite); + + setupCore(); + + await super.runSuite(suite); + + const benchmarks = await extractBenchmarkResults(suite); + if (benchmarks.length === 0) { + console.warn( + `[CodSpeed] No benchmark results found after suite execution`, + ); + return; + } + writeAndLogWalltimeResults(benchmarks); + } + + private populateBenchmarkUris(suite: RunnerTestSuite, parentPath = ""): void { + const currentPath = + parentPath !== "" ? `${parentPath}::${suite.name}` : suite.name; + + for (const task of suite.tasks) { + if (task.type === "suite") { + this.suiteUris.set(task.id, `${currentPath}::${task.name}`); + this.populateBenchmarkUris(task, currentPath); + } + } + } + + private getBenchmarkUri(taskName: string): string { + if (this.currentSuiteId === null) { + throw new Error("currentSuiteId is null - something went wrong"); + } + const suiteUri = this.suiteUris.get(this.currentSuiteId) || ""; + return `${suiteUri}::${taskName}`; + } + + async importTinybench(): Promise { + const tinybench = await super.importTinybench(); + + // `tinybench` is a frozen ES module namespace, so the `Bench` export cannot + // be reassigned. The shared `Task.prototype` is patched in place; the + // instrumented `Bench` is handed back through a fresh module-shaped object + // that Vitest destructures from. + patchTaskRunOnce(tinybench.Task); + + return { + ...tinybench, + Bench: this.createInstrumentedBench(tinybench), + }; + } + + private createInstrumentedBench( + tinybench: Tinybench, + ): typeof tinybench.Bench { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const runner = this; + const OriginalBench = tinybench.Bench; + + class InstrumentedBench extends OriginalBench { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(...benchArgs: any[]) { + super(...benchArgs); + installInstrumentHooks(this as unknown as TinybenchBench, (taskName) => + runner.getBenchmarkUri(taskName), + ); + } + } + + return InstrumentedBench; + } + + // Allow tinybench to retrieve the path to the currently running suite + async onTaskUpdate( + _: RunnerTaskResultPack[], + events: RunnerTaskEventPack[], + ): Promise { + events.map((event) => { + const [id, eventName] = event; + + if (eventName === "suite-prepare") { + this.currentSuiteId = id; + } + }); + } +} + +export default WalltimeRunner; diff --git a/packages/vitest-plugin/src/runner.ts b/packages/vitest-plugin/src/runner.ts deleted file mode 100644 index 60a627a6..00000000 --- a/packages/vitest-plugin/src/runner.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { AnalysisRunner } from "./analysis"; - -export default AnalysisRunner; diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts new file mode 100644 index 00000000..9c5c31c0 --- /dev/null +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -0,0 +1,267 @@ +import { + getGitDir, + getInstrumentMode, + InstrumentHooks, + setupCore, + teardownCore, + type Benchmark, +} from "@codspeed/core"; +import { createRequire } from "module"; +import path from "path"; +import type { + BenchmarkGroup, + BenchmarkProvider, + BenchRegistrationInput, + BenchResult, + BenchRunOptions, +} from "vitest"; +import { + installInstrumentHooks, + rootFrameRegisterFn, + runAnalysisTask, + tinybenchTaskToBenchmark, + writeAndLogWalltimeResults, + type TinybenchBench, + type TinybenchFnOptions, + type TinybenchOptions, + type TinybenchTask, +} from "../instrument"; + +/** tinybench's statistics for one dimension (latency or throughput). */ +type BenchStatistics = BenchResult["latency"]; + +/** + * The subset of the host tinybench Bench the walltime path drives. The timing + * fields are the *resolved* options (tinybench exposes them as readonly + * instance properties), which is what the recorded benchmark config reports. + */ +interface TinybenchWithTasks extends TinybenchBench { + add: ( + name: string, + fn: BenchRegistrationInput["fn"], + fnOpts?: TinybenchFnOptions, + ) => unknown; + run: () => Promise; + tasks: TinybenchTask[]; + readonly iterations: number; + readonly time: number; + readonly warmup: boolean; + readonly warmupIterations: number; + readonly warmupTime: number; +} + +/** + * The host tinybench module, modeled structurally: the plugin's own tinybench + * dependency may be a different major, so its types can't describe the module + * resolved from the installed Vitest. `BenchRunOptions` is Vitest's re-export of + * the tinybench options it accepts, hence of the version it depends on. + */ +interface HostTinybench { + Bench: new (options?: BenchRunOptions) => TinybenchWithTasks; +} + +const isWalltime = getInstrumentMode() === "walltime"; + +// Vitest imports a provider module at most once per worker, and the plugin only +// wires this one up when CodSpeed drives the run (see `V5Backend`), so this is +// the per-worker setup. It must not be redone per benchmark: `setupCore()` +// truncates the process' perf map, which has to cover the whole worker. +setupCore(); +process.once("beforeExit", () => teardownCore()); + +/** + * Build the URI prefix shared by every benchmark of a group: the git-relative + * file path followed by the suite/test path, `::`-separated (e.g. + * `src/a.bench.ts::my suite::my test`). Each registration name is appended to + * it, so a group registering several benchmarks (`bench.compare()`) reports one + * URI per benchmark. + */ +function buildGroupUri(test: BenchmarkGroup["test"]): string { + const filepath = test.file?.filepath; + if (!filepath) { + throw new Error("[CodSpeed] could not resolve the running benchmark file"); + } + const gitDir = getGitDir(filepath); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + const relativeFile = path.relative(gitDir, filepath); + // `fullTestName` uses " > " between suite levels; normalize to "::". + const testPath = test.fullTestName.split(" > ").join("::"); + return [relativeFile, testPath].filter(Boolean).join("::"); +} + +/** + * Resolve the tinybench the *host* Vitest uses so walltime mode returns results + * in the exact shape Vitest serializes. The plugin's own tinybench may be a + * different major, so it is resolved relative to the installed Vitest. + */ +async function importHostTinybench(): Promise { + const require = createRequire(import.meta.url); + const vitestRequire = createRequire(require.resolve("vitest/package.json")); + return import(vitestRequire.resolve("tinybench")); +} + +function analysisResult(name: string): BenchResult { + // Zeroed statistics: enough to satisfy Vitest's reporter and its serializer. + const statistics: BenchStatistics = { + aad: 0, + critical: 0, + df: 0, + mad: 0, + max: 0, + mean: 0, + min: 0, + moe: 0, + p50: 0, + p75: 0, + p99: 0, + p995: 0, + p999: 0, + rme: 0, + samples: undefined, + samplesCount: 0, + sd: 0, + sem: 0, + variance: 0, + }; + + return { + name, + state: "completed", + latency: { ...statistics }, + throughput: { ...statistics }, + period: 0, + totalTime: 0, + runtime: "node", + runtimeVersion: process.versions.node, + timestampProviderName: "codspeed", + }; +} + +/** + * The CodSpeed benchmark provider. Two modes: + * - analysis (instrumentation/simulation): CodSpeed runs each fn itself under a + * tight instrument window and returns zeroed-but-valid results — the real + * measurement is captured by the instrument, not returned here. + * - walltime: tinybench drives the measured loop; CodSpeed brackets it with the + * instrument window and converts each task's stats into a result. + */ +const provider: BenchmarkProvider = { + async run({ test, registrations, options }): Promise { + // Resolve the URI up front, outside any measured window — it walks the + // filesystem (git root lookup), which must not land inside a sample. + const groupUri = buildGroupUri(test); + const getUri = (name: string) => `${groupUri}::${name}`; + + if (isWalltime) { + return runWalltime(registrations, options, getUri, test); + } + return runAnalysis(registrations, getUri); + }, +}; + +async function runAnalysis( + registrations: BenchRegistrationInput[], + getUri: (name: string) => string, +): Promise { + const label = InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; + const results: BenchResult[] = []; + for (const { name, fn, fnOpts } of registrations) { + const uri = getUri(name); + await runAnalysisTask({ fn, fnOpts }, uri); + console.log(`[CodSpeed] ${label} ${uri}`); + results.push(analysisResult(name)); + } + return results; +} + +async function runWalltime( + registrations: BenchRegistrationInput[], + options: BenchRunOptions | undefined, + getUri: (name: string) => string, + test: BenchmarkGroup["test"], +): Promise { + const { Bench } = await importHostTinybench(); + const bench = new Bench({ + signal: test.context.signal, + ...options, + // walltime needs per-iteration samples to compute quantiles + retainSamples: true, + }); + + for (const { name, fn, fnOpts } of registrations) { + // the root frame must be baked into the registered fn (tinybench v6 keeps + // `fn` in a private field, so it can't be wrapped after the fact) + bench.add(name, rootFrameRegisterFn(fn), fnOpts); + } + + installInstrumentHooks(bench, getUri); + const tasks = await bench.run(); + + throwOnErroredTasks(tasks); + collectWalltimeResults(bench, getUri); + + return tasks.map(tinybenchTaskToBenchResult); +} + +/** + * Surface benchmark failures the way Vitest's own provider does: a single error + * is rethrown as is, several are aggregated, and the test fails instead of + * reporting empty results. + */ +function throwOnErroredTasks(tasks: TinybenchTask[]): void { + const errors = tasks + .filter((task) => task.result?.state === "errored") + .map((task) => task.result?.error); + if (errors.length === 1) { + throw errors[0]; + } + if (errors.length > 1) { + throw new AggregateError(errors, "Some benchmarks failed"); + } +} + +function collectWalltimeResults( + bench: TinybenchWithTasks, + getUri: (name: string) => string, +): void { + const options: TinybenchOptions = { + time: bench.time, + iterations: bench.iterations, + // A bench with warmup turned off keeps its warmup timings, so they'd + // otherwise be reported as if a warmup had run. + warmupTime: bench.warmup ? bench.warmupTime : 0, + warmupIterations: bench.warmup ? bench.warmupIterations : 0, + }; + const benchmarks: Benchmark[] = []; + for (const task of bench.tasks) { + if (task.result?.state !== "completed") continue; + const benchmark = tinybenchTaskToBenchmark( + task, + getUri(task.name), + options, + ); + if (benchmark) { + benchmarks.push(benchmark); + } + } + writeAndLogWalltimeResults(benchmarks); +} + +function tinybenchTaskToBenchResult(task: TinybenchTask): BenchResult { + const result = task.result; + if (result?.state !== "completed") { + throw new Error( + `[CodSpeed] benchmark "${task.name}" did not complete: received "${result?.state ?? "no result"}"`, + ); + } + // tinybench's completed result already carries the full statistics surface + // Vitest serializes; pass it through, keyed by the task name. + return { + ...(result as unknown as Omit), + name: task.name, + }; +} + +export default provider; diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts new file mode 100644 index 00000000..d9c707a1 --- /dev/null +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -0,0 +1,146 @@ +import { getInstrumentMode } from "@codspeed/core"; +import { readFileSync } from "fs"; +import { createRequire } from "module"; +import { join } from "path"; +import { type ViteUserConfig } from "vitest/config"; + +/** + * Everything about integrating with Vitest that depends on which Vitest + * generation the user installed, resolved once so the rest of the plugin reads a + * `VitestBackend` and never inspects the version itself. + */ +export interface VitestBackend { + /** + * Whether the plugin should stay active for this Vite `mode`. When false the + * plugin's `apply` returns false and it is dropped entirely. + */ + isActiveForViteMode(mode: string): boolean; + + /** + * Whether the current invocation is running benchmarks (as opposed to tests), + * given the incoming config and Vite `mode`. + */ + isBenchmarkRun(config: ViteUserConfig, mode: string): boolean; + + /** + * The `test` config fragment that wires the benchmark instrumentation into + * Vitest: the V8 exec args (whose placement moved across versions) plus the + * integration seam. Legacy wires a custom runner subclass (and, in walltime + * mode, asks tinybench to retain samples); v5 wires a `benchmark.provider` + * that owns execution and sample retention entirely. + */ + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"]; +} + +/** + * The integration seam differs across Vitest generations: on 3/4 benchmarks run + * through a `NodeBenchmarkRunner` subclass (via `vitest/runners` / + * `vitest/suite`), while 5+ exposes a `benchmark.provider` API and dropped those + * entrypoints. + * + * When the version cannot be detected we assume the latest supported major. + */ +export function resolveVitestBackend(): VitestBackend { + const major = getVitestMajorVersion() ?? 5; + return major >= 5 ? new V5Backend() : new LegacyBackend(major); +} + +/** + * Resolve the major version of the Vitest the *user's project* depends on, not + * the one bundled alongside this plugin. Returns null when it cannot be found, + * letting `resolveVitestBackend` fall back to the latest supported major. + */ +function getVitestMajorVersion(): number | null { + try { + const require = createRequire(join(process.cwd(), "package.json")); + const vitestPkgPath = require.resolve("vitest/package.json"); + const vitestPkg = JSON.parse(readFileSync(vitestPkgPath, "utf-8")); + return parseInt(vitestPkg.version.split(".")[0], 10); + } catch { + return null; + } +} + +/** + * Vitest 5+. There is no dedicated benchmark Vite mode anymore (`vitest bench` + * runs under `"test"`), so the plugin stays active for every mode. + * Instrumentation is installed through a `benchmark.provider` that owns + * benchmark execution (see `v5/provider.ts`). + */ +class V5Backend implements VitestBackend { + isActiveForViteMode(): boolean { + return true; + } + + /** + * Vitest 5 clones its benchmark project from the resolved one *after* the Vite + * config hooks ran, so a benchmark run cannot be read off the incoming config: + * `test.benchmark.enabled` still holds the user's value and `vitest bench` + * only sets an internal CLI flag. CodSpeed exclusively drives benchmark runs, + * so gate on the instrument mode instead. Without CodSpeed the plugin injects + * nothing and Vitest runs the benchmarks through its own tinybench provider. + */ + isBenchmarkRun(): boolean { + return getInstrumentMode() !== "disabled"; + } + + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"] { + return { + execArgv: v8Flags, + // The provider owns benchmark execution: it runs the registered functions + // under instrumentation (analysis) or drives tinybench itself (walltime). + benchmark: { provider: resolveFile("v5/provider") }, + }; + } +} + +/** + * Vitest 3/4. `vitest bench` runs under a dedicated `"benchmark"` Vite mode, and + * instrumentation is installed through a custom `test.runner` subclass of + * `NodeBenchmarkRunner`, one per instrument mode (`analysis` / `walltime`). + */ +class LegacyBackend implements VitestBackend { + constructor(private readonly major: number) {} + + isActiveForViteMode(mode: string): boolean { + return mode === "benchmark"; + } + + isBenchmarkRun(_config: ViteUserConfig, mode: string): boolean { + return mode === "benchmark"; + } + + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"] { + const instrumentMode = getInstrumentMode(); + const runner = + instrumentMode === "disabled" + ? undefined + : resolveFile(join("legacy", instrumentMode)); + + // Walltime asks tinybench to retain per-iteration samples so the runner can + // compute quantiles. On tinybench v2 (Vitest 3/4) the option is + // `includeSamples` (renamed `retainSamples` in v6). + const benchmark = + instrumentMode === "walltime" ? { includeSamples: true } : undefined; + + return { + // Vitest 3 nests exec args under `poolOptions.forks`; v4 moved them to a + // top-level `test.execArgv`. + // See: https://vitest.dev/guide/migration.html#pool-rework + ...(this.major >= 4 + ? { execArgv: v8Flags } + : { poolOptions: { forks: { execArgv: v8Flags } } }), + ...(runner && { runner }), + ...(benchmark && { benchmark }), + } as ViteUserConfig["test"]; + } +} diff --git a/packages/vitest-plugin/src/walltime/index.ts b/packages/vitest-plugin/src/walltime/index.ts deleted file mode 100644 index 880d9279..00000000 --- a/packages/vitest-plugin/src/walltime/index.ts +++ /dev/null @@ -1,216 +0,0 @@ -import { - InstrumentHooks, - MARKER_TYPE_BENCHMARK_END, - MARKER_TYPE_BENCHMARK_START, - setupCore, - wrapWithRootFrame, - writeWalltimeResults, -} from "@codspeed/core"; -import type * as tinybench from "tinybench"; -import { - RunnerTaskEventPack, - RunnerTaskResultPack, - type RunnerTestSuite, -} from "vitest"; -import { patchRootSuiteWithFullFilePath } from "../common"; -import { NodeBenchmarkRunner } from "../compat"; -import { extractBenchmarkResults } from "./utils"; - -type Tinybench = typeof tinybench; - -/** A tinybench task, exposing the `fn` the runner wraps with the root frame. */ -interface TinybenchTask { - name: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - fn: (...args: any[]) => any; -} - -/** tinybench's per-task setup/teardown hook signature. */ -type TinybenchHook = ( - task: TinybenchTask, - mode: "run" | "warmup", -) => Promise | void; - -/** The mutable subset of a tinybench Bench the runner reaches into. */ -interface TinybenchBench { - setup: TinybenchHook; - teardown: TinybenchHook; -} - -/** - * WalltimeRunner uses Vitest's default benchmark execution - * and extracts results from the suite after completion - */ -export class WalltimeRunner extends NodeBenchmarkRunner { - private isTinybenchHookedWithCodspeed = false; - private suiteUris = new Map(); - /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks - private currentSuiteId: string | null = null; - // Carries the window start timestamp from the setup hook to the teardown - // hook. Tasks run strictly sequentially, so a single field is enough. - private runStart: bigint | null = null; - - async runSuite(suite: RunnerTestSuite): Promise { - patchRootSuiteWithFullFilePath(suite); - this.populateBenchmarkUris(suite); - - setupCore(); - - await super.runSuite(suite); - - const benchmarks = await extractBenchmarkResults(suite); - - if (benchmarks.length > 0) { - writeWalltimeResults(benchmarks); - console.log( - `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.`, - ); - } else { - console.warn( - `[CodSpeed] No benchmark results found after suite execution`, - ); - } - } - - private populateBenchmarkUris(suite: RunnerTestSuite, parentPath = ""): void { - const currentPath = - parentPath !== "" ? `${parentPath}::${suite.name}` : suite.name; - - for (const task of suite.tasks) { - if (task.type === "suite") { - this.suiteUris.set(task.id, `${currentPath}::${task.name}`); - this.populateBenchmarkUris(task, currentPath); - } - } - } - - private getBenchmarkUri(taskName: string): string { - if (this.currentSuiteId === null) { - throw new Error("currentSuiteId is null - something went wrong"); - } - const suiteUri = this.suiteUris.get(this.currentSuiteId) || ""; - return `${suiteUri}::${taskName}`; - } - - async importTinybench(): Promise { - const tinybench = await super.importTinybench(); - - // `tinybench` is a frozen ES module namespace, so the `Bench` export cannot - // be reassigned. Mutating the shared `Task.prototype` in place is allowed - // and only needs to happen once; the instrumented `Bench` is handed back - // through a fresh module-shaped object that Vitest destructures from. - if (!this.isTinybenchHookedWithCodspeed) { - this.isTinybenchHookedWithCodspeed = true; - this.patchTaskWithRootFrame(tinybench); - } - - return { - ...tinybench, - Bench: this.createInstrumentedBench(tinybench), - }; - } - - /** - * Wrap each task's function with the root frame so collected stacks can be - * attributed to a benchmark. The window itself is driven by the bench's - * setup/teardown hooks (see createInstrumentedBench). - */ - private patchTaskWithRootFrame(tinybench: Tinybench): void { - const originalRun = tinybench.Task.prototype.run; - - tinybench.Task.prototype.run = async function () { - const task = this as unknown as TinybenchTask; - const originalFn = task.fn; - task.fn = wrapWithRootFrame(() => originalFn.call(task)); - - try { - await originalRun.call(this); - } finally { - task.fn = originalFn; - } - - return this; - }; - } - - /** - * Drive the instrumentation window from each bench's run-mode setup/teardown - * hooks so it brackets only tinybench's measured loop, excluding the warmup - * that Vitest runs beforehand and the statistics computation tinybench - * performs after the loop. Wrapping the whole `Task.run()` would otherwise - * fold all of that framework overhead into the recorded sample. - * - * User-provided hooks are preserved and keep their order relative to the work - * under test. - */ - private createInstrumentedBench( - tinybench: Tinybench, - ): typeof tinybench.Bench { - // eslint-disable-next-line @typescript-eslint/no-this-alias - const runner = this; - const OriginalBench = tinybench.Bench; - - class InstrumentedBench extends OriginalBench { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - constructor(...benchArgs: any[]) { - super(...benchArgs); - runner.installInstrumentHooks(this as unknown as TinybenchBench); - } - } - - return InstrumentedBench; - } - - private installInstrumentHooks(bench: TinybenchBench): void { - const userSetup = bench.setup; - const userTeardown = bench.teardown; - - bench.setup = async (task, mode) => { - await userSetup(task, mode); - if (mode === "run") { - InstrumentHooks.startBenchmark(); - this.runStart = InstrumentHooks.currentTimestamp(); - } - }; - - bench.teardown = async (task, mode) => { - if (mode === "run") { - this.closeInstrumentWindow(this.getBenchmarkUri(task.name)); - } - await userTeardown(task, mode); - }; - } - - private closeInstrumentWindow(uri: string): void { - const runEnd = InstrumentHooks.currentTimestamp(); - const pid = process.pid; - - // Benchmark markers must land inside the sample window opened by - // startBenchmark(), so they have to be emitted before stopBenchmark() - // closes it. The runner consumes the FIFO stream in order, so a marker - // sent after StopBenchmark falls outside the sample and breaks the - // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, this.runStart!); - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); - - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(pid, uri); - this.runStart = null; - } - - // Allow tinybench to retrieve the path to the currently running suite - async onTaskUpdate( - _: RunnerTaskResultPack[], - events: RunnerTaskEventPack[], - ): Promise { - events.map((event) => { - const [id, eventName] = event; - - if (eventName === "suite-prepare") { - this.currentSuiteId = id; - } - }); - } -} - -export default WalltimeRunner; diff --git a/packages/vitest-plugin/src/walltime/utils.ts b/packages/vitest-plugin/src/walltime/utils.ts deleted file mode 100644 index e0e3313e..00000000 --- a/packages/vitest-plugin/src/walltime/utils.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { - calculateQuantiles, - msToNs, - msToS, - type Benchmark, - type BenchmarkStats, -} from "@codspeed/core"; -import { - type RunnerTaskResult, - type RunnerTestSuite, - type Benchmark as VitestBenchmark, -} from "vitest"; -import { isVitestTaskBenchmark } from "../common"; -import { getBenchOptions } from "../compat"; - -export async function extractBenchmarkResults( - suite: RunnerTestSuite, - parentPath = "", -): Promise { - const benchmarks: Benchmark[] = []; - const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; - - for (const task of suite.tasks) { - if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { - const benchmark = await processBenchmarkTask(task, currentPath); - if (benchmark) { - benchmarks.push(benchmark); - } - } else if (task.type === "suite") { - const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); - benchmarks.push(...nestedBenchmarks); - } - } - - return benchmarks; -} - -async function processBenchmarkTask( - task: VitestBenchmark, - suitePath: string, -): Promise { - const uri = `${suitePath}::${task.name}`; - - const result = task.result; - if (!result) { - console.warn(` ⚠ No result data available for ${uri}`); - return null; - } - - try { - // Get tinybench configuration options from vitest - const benchOptions = getBenchOptions(task); - - const stats = convertVitestResultToBenchmarkStats(result, benchOptions); - - if (stats === null) { - console.log(` ✔ No walltime data to collect for ${uri}`); - return null; - } - - const coreBenchmark: Benchmark = { - name: task.name, - uri, - config: { - max_rounds: benchOptions.iterations ?? null, - max_time_ns: benchOptions.time ? msToNs(benchOptions.time) : null, - min_round_time_ns: null, // tinybench does not have an option for this - warmup_time_ns: - benchOptions.warmupIterations !== 0 && benchOptions.warmupTime - ? msToNs(benchOptions.warmupTime) - : null, - }, - stats, - }; - - console.log(` ✔ Collected walltime data for ${uri}`); - return coreBenchmark; - } catch (error) { - console.warn(` ⚠ Failed to process benchmark result for ${uri}:`, error); - return null; - } -} - -function convertVitestResultToBenchmarkStats( - result: RunnerTaskResult, - benchOptions: { - time?: number; - warmupTime?: number; - warmupIterations?: number; - iterations?: number; - }, -): BenchmarkStats | null { - const benchmark = result.benchmark; - - if (!benchmark) { - throw new Error("No benchmark data available in result"); - } - - const { totalTime, min, max, mean, sd, samples } = benchmark; - - // Get individual sample times in nanoseconds and sort them - const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); - const meanNs = msToNs(mean); - const stdevNs = msToNs(sd); - - if (sortedTimesNs.length == 0) { - // Sometimes the benchmarks can be completely optimized out and not even run, but its beforeEach and afterEach hooks are still executed, and the task is still considered a success. - // This is the case for the hooks.bench.ts example in this package - return null; - } - - const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = - calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); - - return { - min_ns: msToNs(min), - max_ns: msToNs(max), - mean_ns: meanNs, - stdev_ns: stdevNs, - q1_ns, - median_ns, - q3_ns, - total_time: msToS(totalTime), - iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration - rounds: sortedTimesNs.length, - iqr_outlier_rounds, - stdev_outlier_rounds, - warmup_iters: benchOptions.warmupIterations ?? 0, - }; -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d6e150dd..040c2b91 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -255,6 +255,30 @@ importers: specifier: ^3.2.4 version: 3.2.4(@types/node@24.13.3)(lightningcss@1.33.0)(tsx@4.23.12)(yaml@2.9.0) + examples/with-vitest-v4: + devDependencies: + '@codspeed/vitest-plugin': + specifier: workspace:* + version: link:../../packages/vitest-plugin + typescript: + specifier: ^5.1.3 + version: 5.8.3 + vitest: + specifier: ^4.1.9 + version: 4.1.11(@types/node@24.13.3)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + + examples/with-vitest-v5: + devDependencies: + '@codspeed/vitest-plugin': + specifier: workspace:* + version: link:../../packages/vitest-plugin + typescript: + specifier: ^5.1.3 + version: 5.8.3 + vitest: + specifier: ^5.0.0 + version: 5.0.0(@types/node@24.13.3)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + packages/benchmark.js-plugin: dependencies: '@codspeed/core': @@ -368,8 +392,8 @@ importers: specifier: ^8.0.0 version: 8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) vitest: - specifier: ^4.1.11 - version: 4.1.11(@types/node@24.13.3)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + specifier: ^5.0.0 + version: 5.0.0(@types/node@24.13.3)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) packages: @@ -1831,8 +1855,8 @@ packages: '@jridgewell/trace-mapping@0.3.18': resolution: {integrity: sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==} - '@jridgewell/trace-mapping@0.3.29': - resolution: {integrity: sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==} + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} @@ -2762,6 +2786,17 @@ packages: vite: optional: true + '@vitest/mocker@5.0.0': + resolution: {integrity: sha512-66PGTMIiVJP3t4a5yxU9qPtf7MdTBs8jmToMvy+HVflB3Yy13WJZTtPePdvU+wjRV02SKK5doLbSA6o9pwOmiA==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + '@vitest/pretty-format@3.2.4': resolution: {integrity: sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==} @@ -2798,6 +2833,9 @@ packages: '@vitest/spy@4.1.11': resolution: {integrity: sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==} + '@vitest/spy@5.0.0': + resolution: {integrity: sha512-uy+luWBAPw9XfthoHi5AkfHUnuPYEESjl0p/r+meoBnU8bxg5GDQ3Ey8MjcJ6sqahkL4PFyrvfMJJBw7LbU06g==} + '@vitest/utils@3.2.4': resolution: {integrity: sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==} @@ -5119,6 +5157,9 @@ packages: resolution: {integrity: sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==} engines: {node: '>=12'} + magic-string@1.2.3: + resolution: {integrity: sha512-Bpb0W2TbLKOZ7vJnOUnVRGq3WL2p+ISV29M6hYPL1AFCpyKZpdr5ytiXoTSSxRVhg8YW7f65+6gbG8WG6PCa/g==} + make-dir@2.1.0: resolution: {integrity: sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==} engines: {node: '>=6'} @@ -5498,6 +5539,10 @@ packages: obug@2.1.1: resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + obug@2.1.4: + resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} + engines: {node: '>=12.20.0'} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -6387,6 +6432,10 @@ packages: resolution: {integrity: sha512-FlHoQpcFvCzeXK5kVPvV7IVgW/hs/B36QWTz876iSdeJguBDfdTSRQmYmaHX+fQNt4hp+gEFB2XXw+8hT4/y8A==} engines: {node: '>=20.0.0'} + tinybench@6.1.4: + resolution: {integrity: sha512-9APumHG7r4yOk4X4WlkmE71aZcv1gvin1czO3OQ1U9iJcFA5Ja/ygyb0vPOVHTthFozUYs8CLoLUlM8grb2lTQ==} + engines: {node: '>=20.0.0'} + tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} @@ -6394,6 +6443,10 @@ packages: resolution: {integrity: sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==} engines: {node: '>=18'} + tinyexec@1.3.0: + resolution: {integrity: sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==} + engines: {node: '>=18'} + tinyglobby@0.2.12: resolution: {integrity: sha512-qkf4trmKSIiMTs/E63cxH+ojC2unam7rJ0WrauAzpT3ECNTxGRMlaXxVbfxMUC/w0LaYk6jQ4y/nGR9uBO3tww==} engines: {node: '>=12.0.0'} @@ -6872,6 +6925,47 @@ packages: jsdom: optional: true + vitest@5.0.0: + resolution: {integrity: sha512-gpsMNoRhMjMktVxPtstOH4/PJuPyovVaMDr4oDilXaGH1EcqM2OE96SoHT2VIQ6fTGtTjqmHDrEu2X9RQiXf8Q==} + engines: {node: ^22.12.0 || ^24.0.0 || >=26.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 5.0.0 + '@vitest/browser-preview': 5.0.0 + '@vitest/browser-webdriverio': ^5.0.0-beta.5 || >=5.0.0 + '@vitest/coverage-istanbul': 5.0.0 + '@vitest/coverage-v8': 5.0.0 + '@vitest/ui': 5.0.0 + happy-dom: '*' + jsdom: '*' + vite: ^6.4.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + walk-up-path@3.0.1: resolution: {integrity: sha512-9YlCL/ynK3CTlrSRrDxZvUauLzAswPCrsaCgilqFevUYpeEW0/3ScEjaa3kbW/T0ghhkEr7mv+fpjqn1Y1YuTA==} @@ -7028,7 +7122,7 @@ snapshots: '@ampproject/remapping@2.3.0': dependencies: '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@apidevtools/json-schema-ref-parser@9.0.9': dependencies: @@ -7106,7 +7200,7 @@ snapshots: '@babel/parser': 7.28.0 '@babel/types': 7.28.1 '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 jsesc: 3.1.0 '@babel/helper-annotate-as-pure@7.22.5': @@ -8655,7 +8749,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@types/node': 24.13.3 chalk: 4.1.2 collect-v8-coverage: 1.0.2 @@ -8693,7 +8787,7 @@ snapshots: '@jest/source-map@29.6.3': dependencies: - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 callsites: 3.1.0 graceful-fs: 4.2.11 @@ -8749,7 +8843,7 @@ snapshots: dependencies: '@babel/core': 7.28.0 '@jest/types': 29.6.3 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 babel-plugin-istanbul: 6.1.1 chalk: 4.1.2 convert-source-map: 2.0.0 @@ -8786,7 +8880,7 @@ snapshots: '@jridgewell/gen-mapping@0.3.12': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@jridgewell/gen-mapping@0.3.3': dependencies: @@ -8815,7 +8909,7 @@ snapshots: '@jridgewell/resolve-uri': 3.1.0 '@jridgewell/sourcemap-codec': 1.4.14 - '@jridgewell/trace-mapping@0.3.29': + '@jridgewell/trace-mapping@0.3.31': dependencies: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 @@ -9800,6 +9894,15 @@ snapshots: optionalDependencies: vite: 8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) + '@vitest/mocker@5.0.0(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0))': + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + '@vitest/spy': 5.0.0 + estree-walker: 3.0.3 + magic-string: 1.2.3 + optionalDependencies: + vite: 8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) + '@vitest/pretty-format@3.2.4': dependencies: tinyrainbow: 2.0.0 @@ -9855,6 +9958,8 @@ snapshots: '@vitest/spy@4.1.11': {} + '@vitest/spy@5.0.0': {} + '@vitest/utils@3.2.4': dependencies: '@vitest/pretty-format': 3.2.4 @@ -12887,6 +12992,10 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.4.15 + magic-string@1.2.3: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + make-dir@2.1.0: dependencies: pify: 4.0.1 @@ -13354,6 +13463,8 @@ snapshots: obug@2.1.1: {} + obug@2.1.4: {} + once@1.4.0: dependencies: wrappy: 1.0.2 @@ -14304,10 +14415,14 @@ snapshots: tinybench@6.0.2: {} + tinybench@6.1.4: {} + tinyexec@0.3.2: {} tinyexec@1.0.2: {} + tinyexec@1.3.0: {} + tinyglobby@0.2.12: dependencies: fdir: 6.5.0(picomatch@4.0.7) @@ -14593,7 +14708,7 @@ snapshots: v8-to-istanbul@9.3.0: dependencies: - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@types/istanbul-lib-coverage': 2.0.6 convert-source-map: 2.0.0 @@ -14759,6 +14874,27 @@ snapshots: transitivePeerDependencies: - msw + vitest@5.0.0(@types/node@24.13.3)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)): + dependencies: + '@types/chai': 5.2.2 + '@vitest/mocker': 5.0.0(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + chai: 6.2.2 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 1.2.3 + obug: 2.1.4 + picomatch: 4.0.7 + std-env: 4.2.0 + tinybench: 6.1.4 + tinyexec: 1.3.0 + tinyglobby: 0.2.17 + vite: 8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 24.13.3 + transitivePeerDependencies: + - msw + walk-up-path@3.0.1: {} walker@1.0.8: