Support vitest 5

.gitignore

@@ -132,3 +132,5 @@ packages/app/.env
 # turbo
 .turbo/
 .rollup.cache/
+
+.codspeed

CLAUDE.md

@@ -94,4 +94,25 @@ Based on the codebase analysis, to add stats access features:
   - Build: `pnpm turbo run build --filter=<package-name>`
   - Typecheck: `pnpm turbo run typecheck --filter=<package-name>`
   - Lint: `pnpm turbo run lint --filter=<package-name>`
-  - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`).
+  - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`).
+
+## Testing a plugin during development
+
+A plugin behaves differently depending on whether CodSpeed is driving the run,
+so exercise all three of these when developing or reviewing a plugin change
+(build the plugin first — the benches import from `dist`):
+
+1. **Fallback (not under CodSpeed).** No env vars. The plugin must stay out of
+   the way and let the framework run its benchmarks normally (no instrumentation,
+   no hijacked output). e.g. `pnpm turbo run bench --filter=<package-name>`.
+2. **Instrumentation / simulation.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=simulation`
+   (or `instrumentation`). The plugin hijacks the run to do a single instrumented
+   pass per benchmark and prints `Measured/Checked <uri>` instead of the normal
+   harness output.
+3. **Walltime.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=walltime`. The plugin
+   instruments the framework's real benchmark loop and collects walltime results.
+
+Running these locally outside the CodSpeed runner is expected to log
+`instrument-hooks: failed to write environment.json` and skip actual measurement
+writes — the point is to verify the plugin's control flow and output per mode,
+not to produce real measurements.

examples/with-vitest-v4/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "with-vitest-v4",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "bench-vitest": "vitest bench --run"
+  },
+  "devDependencies": {
+    "@codspeed/vitest-plugin": "workspace:*",
+    "typescript": "^5.1.3",
+    "vitest": "^4.1.9"
+  }
+}

examples/with-vitest-v4/src/fibonacci.bench.ts

@@ -0,0 +1,20 @@
+import { bench, describe } from "vitest";
+import { iterativeFibonacci, recursiveFibonacci } from "./fibonacci";
+
+describe("fibonacci", () => {
+  bench("recursive fibo 15", () => {
+    recursiveFibonacci(15);
+  });
+
+  bench("recursive fibo 20", () => {
+    recursiveFibonacci(20);
+  });
+
+  bench("iterative fibo 15", () => {
+    iterativeFibonacci(15);
+  });
+
+  bench("iterative fibo 20", () => {
+    iterativeFibonacci(20);
+  });
+});

examples/with-vitest-v4/src/fibonacci.ts

@@ -0,0 +1,17 @@
+export function recursiveFibonacci(n: number): number {
+  if (n < 2) {
+    return n;
+  }
+  return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2);
+}
+
+export function iterativeFibonacci(n: number): number {
+  let a = 0;
+  let b = 1;
+  for (let i = 0; i < n; i++) {
+    const temp = a + b;
+    a = b;
+    b = temp;
+  }
+  return a;
+}

examples/with-vitest-v4/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "lib": ["es2023"],
+    "module": "ESNext",
+    "verbatimModuleSyntax": true,
+    "target": "es2022",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "moduleResolution": "Node"
+  }
+}

examples/with-vitest-v4/vitest.config.ts

@@ -0,0 +1,6 @@
+import codspeedPlugin from "@codspeed/vitest-plugin";
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  plugins: [codspeedPlugin()],
+});

examples/with-vitest-v5/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "with-vitest-v5",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "bench-vitest": "vitest bench --run"
+  },
+  "devDependencies": {
+    "@codspeed/vitest-plugin": "workspace:*",
+    "typescript": "^5.1.3",
+    "vitest": "^5.0.0"
+  }
+}

examples/with-vitest-v5/src/fibonacci.bench.ts

@@ -0,0 +1,38 @@
+import { describe, test } from "vitest";
+import {
+  iterativeFibonacci as iterativeFibonacciExport,
+  recursiveFibonacci as recursiveFibonacciExport,
+} from "./fibonacci";
+
+// Read the imported bindings once: Vite's module runner exposes exports through
+// getters, and reading one inside the measured loop adds overhead to every
+// iteration (Vitest warns about it).
+const recursiveFibonacci = recursiveFibonacciExport;
+const iterativeFibonacci = iterativeFibonacciExport;
+
+// Vitest 5 declares benchmarks through the `bench` test-context fixture: each
+// `bench()` returns a registration that runs on `.run()`, or as part of a
+// `bench.compare()` group.
+describe("fibonacci", () => {
+  test("fibo 15", async ({ bench }) => {
+    await bench.compare(
+      bench("recursive", () => {
+        recursiveFibonacci(15);
+      }),
+      bench("iterative", () => {
+        iterativeFibonacci(15);
+      }),
+    );
+  });
+
+  test("fibo 20", async ({ bench }) => {
+    await bench.compare(
+      bench("recursive", () => {
+        recursiveFibonacci(20);
+      }),
+      bench("iterative", () => {
+        iterativeFibonacci(20);
+      }),
+    );
+  });
+});

examples/with-vitest-v5/src/fibonacci.ts

@@ -0,0 +1,17 @@
+export function recursiveFibonacci(n: number): number {
+  if (n < 2) {
+    return n;
+  }
+  return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2);
+}
+
+export function iterativeFibonacci(n: number): number {
+  let a = 0;
+  let b = 1;
+  for (let i = 0; i < n; i++) {
+    const temp = a + b;
+    a = b;
+    b = temp;
+  }
+  return a;
+}

examples/with-vitest-v5/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "lib": ["es2023"],
+    "module": "ESNext",
+    "verbatimModuleSyntax": true,
+    "target": "es2022",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "moduleResolution": "Node"
+  }
+}

examples/with-vitest-v5/vitest.config.ts

@@ -0,0 +1,6 @@
+import codspeedPlugin from "@codspeed/vitest-plugin";
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  plugins: [codspeedPlugin()],
+});

packages/vitest-plugin/README.md

@@ -41,10 +41,10 @@ pnpm add --save-dev @codspeed/vitest-plugin vitest
 
 ## Usage
 
-Let's create a fibonacci function and benchmark it with `vitest.bench`:
+Let's create a fibonacci function and benchmark it with Vitest's `bench` fixture:
 
 ```ts title="benches/fibo.bench.ts"
-import { describe, bench } from "vitest";
+import { describe, test } from "vitest";
 
 function fibonacci(n: number): number {
   if (n < 2) {
@@ -54,16 +54,33 @@ function fibonacci(n: number): number {
 }
 
 describe("fibonacci", () => {
-  bench("fibonacci10", () => {
-    fibonacci(10);
-  });
-
-  bench("fibonacci15", () => {
-    fibonacci(15);
+  test("depth", async ({ bench }) => {
+    await bench.compare(
+      bench("fibonacci10", () => {
+        fibonacci(10);
+      }),
+      bench("fibonacci15", () => {
+        fibonacci(15);
+      }),
+    );
   });
 });
 ```
 
+> [!NOTE]
+> The `bench` fixture is a Vitest 5 API. On Vitest 3 and 4, benchmarks are
+> declared with the top-level `bench()` export instead:
+>
+> ```ts
+> import { bench, describe } from "vitest";
+>
+> describe("fibonacci", () => {
+>   bench("fibonacci10", () => {
+>     fibonacci(10);
+>   });
+> });
+> ```
+
 Create or update your `vitest.config.ts` file to use the CodSpeed runner:
 
 ```ts title="vitest.config.ts"
@@ -80,7 +97,6 @@ Finally, run your benchmarks (here with `pnpm`):
 
 ```bash
 $ pnpm vitest bench --run
-[CodSpeed] bench detected but no instrumentation found, falling back to default vitest runner
 
 ... Regular `vitest bench` output
 ```

packages/vitest-plugin/benches/flat.bench.ts

@@ -1,5 +1,10 @@
-import { bench, describe } from "vitest";
-import parsePr from "./parsePr";
+import { describe, test } from "vitest";
+import parsePrExport from "./parsePr";
+
+// Read the imported binding once: Vite's module runner exposes exports through
+// getters, and reading one inside the measured loop adds overhead to every
+// iteration (Vitest warns about it).
+const parsePr = parsePrExport;
 
 const LONG_BODY =
   new Array(1_000)
@@ -9,12 +14,15 @@ const LONG_BODY =
     .join("\n") + "fixes #123";
 
 describe("parsePr", () => {
-  bench("short body", () => {
-    parsePr({ body: "fixes #123", title: "test-1", number: 1 });
-  });
-
-  bench("long body", () => {
-    parsePr({ body: LONG_BODY, title: "test-2", number: 2 });
+  test("body size", async ({ bench }) => {
+    await bench.compare(
+      bench("short body", () => {
+        parsePr({ body: "fixes #123", title: "test-1", number: 1 });
+      }),
+      bench("long body", () => {
+        parsePr({ body: LONG_BODY, title: "test-2", number: 2 });
+      }),
+    );
   });
 });
 
@@ -24,10 +32,13 @@ function fibo(n: number): number {
 }
 
 describe("fibo", () => {
-  bench("fibo 10", () => {
-    fibo(10);
-  });
-  bench("fibo 15", () => {
-    fibo(15);
+  test("depth", async ({ bench }) => {
+    await bench("fibo 10", () => {
+      fibo(10);
+    }).run();
+
+    await bench("fibo 15", () => {
+      fibo(15);
+    }).run();
   });
 });

packages/vitest-plugin/benches/hooks.bench.ts

@@ -1,37 +1,39 @@
-import {
-  afterAll,
-  afterEach,
-  beforeAll,
-  beforeEach,
-  bench,
-  describe,
-  expect,
-} from "vitest";
+import { describe, expect, test } from "vitest";
 
+// Exercises tinybench's per-benchmark hooks, which Vitest 5 exposes through the
+// `bench(name, options, fn)` options object (`beforeAll`/`beforeEach`/...).
 describe("hooks", () => {
   let count = 0;
-  describe("run", () => {
-    beforeAll(() => {
+
+  const hooks = {
+    beforeAll: () => {
       count += 10;
-    });
-    beforeEach(() => {
+    },
+    beforeEach: () => {
       count += 1;
-    });
-    afterEach(() => {
+    },
+    afterEach: () => {
       count -= 1;
-    });
-    afterAll(() => {
+    },
+    afterAll: () => {
       count -= 10;
-    });
+    },
+  };
 
-    bench("one", () => {
-      expect(count).toBe(11);
-    });
-    bench("two", () => {
-      expect(count).toBe(11);
-    });
+  test("hooked benches", async ({ bench }) => {
+    await bench.compare(
+      bench("one", hooks, () => {
+        expect(count).toBe(11);
+      }),
+      bench("two", hooks, () => {
+        expect(count).toBe(11);
+      }),
+    );
   });
-  bench("end", () => {
-    expect(count).toBe(0);
+
+  test("after the hooked benches", async ({ bench }) => {
+    await bench("count is back to zero", () => {
+      expect(count).toBe(0);
+    }).run();
   });
 });

packages/vitest-plugin/benches/macos.bench.ts

@@ -1,4 +1,4 @@
-import { bench, describe } from "vitest";
+import { describe, test } from "vitest";
 
 const isMacOS = process.platform === "darwin";
 
@@ -10,7 +10,9 @@ function fibo(n: number): number {
 // macOS-only benchmark: skipped on every other platform, so it only runs on
 // the `codspeed-walltime-macos` CI job (see .github/workflows/codspeed.yml).
 describe.skipIf(!isMacOS)("macos only", () => {
-  bench("fibo darwin", () => {
-    fibo(30);
+  test("fibo", async ({ bench }) => {
+    await bench("fibo darwin", () => {
+      fibo(30);
+    }).run();
   });
 });