From 263b84c22751ea7b2b458ab7b0445fb0507250dd Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 9 Nov 2026 15:03:31 +0100 Subject: [PATCH 1/4] fix(core): stop V8 incremental marking tasks in analysis mode --predictable and --predictable-gc-schedule fix the heap growing schedule but V8 still posts incremental marking steps as tasks on the event loop. Async work before a benchmark (Vitest's ESM loading, an awaited readFile loop) runs a varying number of those tasks, so when marking finishes, how many GCs ran and where objects end up differ between identical runs. Under Valgrind this shows as steps in both instruction counts (lazy sweeping inside the measured call) and cache misses. --no-incremental-marking-task makes marking progress only on allocation, which is deterministic. The flag exists on every supported Node major (22, 24, 26). --- packages/core/src/introspection.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/core/src/introspection.ts b/packages/core/src/introspection.ts index 8906c1d9..eeb26010 100644 --- a/packages/core/src/introspection.ts +++ b/packages/core/src/introspection.ts @@ -33,6 +33,10 @@ export const getV8Flags = () => { "--predictable-gc-schedule", "--expose-gc", "--no-concurrent-sweeping", + // --predictable keeps posting incremental marking steps as tasks on + // the event loop, so async work before the benchmark (e.g. module + // loading) shifts the GC count and heap layout it measures in. + "--no-incremental-marking-task", "--max-old-space-size=4096", ], ); From 76f84527d151432a2000831b57ee1017109edc37 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 9 Nov 2026 15:03:32 +0100 Subject: [PATCH 2/4] fix(tinybench): keep runSync warmup calls between their own hooks In analysis mode, runSync passed an async lambda to optimizeFunctionSync. Each warmup call returned at its first await, so its afterEach hook ran after the measured call instead of right after the warmup call, and the measured call ran with every warmup's state still live. The lambda is now synchronous, like the function runSync accepts. --- packages/tinybench-plugin/src/analysis.ts | 4 +- .../tests/index.integ.test.ts | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 65bd6183..6c3fe826 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -62,9 +62,9 @@ class AnalysisBenchRunner extends BaseBenchRunner { fnOpts?.beforeAll?.call(task, "run"); - optimizeFunctionSync(async () => { + optimizeFunctionSync(() => { fnOpts?.beforeEach?.call(task, "run"); - await fn(); + fn(); fnOpts?.afterEach?.call(task, "run"); }); diff --git a/packages/tinybench-plugin/tests/index.integ.test.ts b/packages/tinybench-plugin/tests/index.integ.test.ts index 5e71eeb2..3b0ec727 100644 --- a/packages/tinybench-plugin/tests/index.integ.test.ts +++ b/packages/tinybench-plugin/tests/index.integ.test.ts @@ -24,6 +24,9 @@ const mockCore = vi.hoisted(() => { .mockImplementation(async (fn: () => Promise) => { await fn(); }), + optimizeFunctionSync: vi.fn().mockImplementation((fn: () => void) => { + fn(); + }), setupCore: vi.fn(), teardownCore: vi.fn(), writeWalltimeResults: vi.fn(), @@ -208,6 +211,42 @@ describe("Benchmark.Suite", () => { expect(afterAll).toHaveBeenCalledTimes(2); }); + it("should run each runSync call between its own hooks", () => { + mockCore.InstrumentHooks.isInstrumented.mockReturnValue(true); + mockCore.optimizeFunctionSync.mockImplementation((fn: () => void) => { + fn(); + }); + const calls: string[] = []; + const bench = withCodSpeed(new Bench()).add( + "RegExp", + () => { + calls.push("fn"); + }, + { + beforeEach: () => { + calls.push("beforeEach"); + }, + afterEach: () => { + calls.push("afterEach"); + }, + }, + ); + // tinybench calls the function once when it's added + calls.length = 0; + + bench.runSync(); + + // the optimization run, then the measured run + expect(calls).toEqual([ + "beforeEach", + "fn", + "afterEach", + "beforeEach", + "fn", + "afterEach", + ]); + }); + it("should call setupCore and teardownCore only once", async () => { mockCore.InstrumentHooks.isInstrumented.mockReturnValue(true); const bench = withCodSpeed(new Bench()) From 6217b108d8040f155089c40630fb99731e3194fb Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 9 Nov 2026 15:03:33 +0100 Subject: [PATCH 3/4] fix(vitest): await benchmarks that return a promise in simulation The Vitest 5 provider only awaited functions declared async. A function that returns a promise without being async was run on the sync path, so its asynchronous work ran after the measured window closed and after afterEach. The provider now calls a non-async function once between its hooks, like tinybench does, and takes the async path when it returns a thenable. beforeAll is awaited up front for both paths. --- .../src/__tests__/analysis.test.ts | 62 +++++++++++++++++++ packages/vitest-plugin/src/v5/provider.ts | 25 +++++++- .../tests/fixtures/analysis/promise.bench.ts | 24 +++++++ .../tests/fixtures/analysis/vitest.config.ts | 12 ++++ 4 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 packages/vitest-plugin/src/__tests__/analysis.test.ts create mode 100644 packages/vitest-plugin/tests/fixtures/analysis/promise.bench.ts create mode 100644 packages/vitest-plugin/tests/fixtures/analysis/vitest.config.ts diff --git a/packages/vitest-plugin/src/__tests__/analysis.test.ts b/packages/vitest-plugin/src/__tests__/analysis.test.ts new file mode 100644 index 00000000..f31981bc --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/analysis.test.ts @@ -0,0 +1,62 @@ +import { spawn } from "child_process"; +import { once } from "events"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs"; +import { createRequire } from "module"; +import { tmpdir } from "os"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const FIXTURE_DIR = fileURLToPath( + new URL("../../tests/fixtures/analysis", import.meta.url), +); +const VITEST_BIN = join( + dirname(createRequire(import.meta.url).resolve("vitest/package.json")), + "vitest.mjs", +); +const RUN_TIMEOUT_MS = 180_000; + +let tmpDir: string; + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "codspeed-analysis-")); +}); + +afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); +}); + +describe("simulation", () => { + it( + "awaits a function that returns a promise before its afterEach hook", + async () => { + const logFile = join(tmpDir, "events.log"); + writeFileSync(logFile, ""); + // Not spawnSync: blocking this worker times out its calls to Vitest. + const child = spawn(process.execPath, [VITEST_BIN, "bench", "--run"], { + cwd: FIXTURE_DIR, + env: { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + CODSPEED_PROFILE_FOLDER: join(tmpDir, "profile"), + ANALYSIS_LOG_FILE: logFile, + }, + timeout: RUN_TIMEOUT_MS, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + child.stderr?.on("data", (chunk) => (stderr += chunk)); + const [status] = (await once(child, "close")) as [number | null]; + + expect(status, stderr).toBe(0); + // Every call, the measured one included, finishes its work in between + // its own hooks. + const events = readFileSync(logFile, "utf8").trim().split("\n"); + expect(events.join(" ")).toMatch( + /^beforeEach done afterEach( beforeEach done afterEach)*$/, + ); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index 9ce135ad..bc9aeb7c 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -180,8 +180,11 @@ async function runAnalysisTask( fnOpts: TinybenchFnOptions | undefined, uri: string, ): Promise { - if (fn.constructor?.name === "AsyncFunction") { - await fnOpts?.beforeAll?.("run"); + await fnOpts?.beforeAll?.("run"); + if ( + fn.constructor?.name === "AsyncFunction" || + (await returnsPromise(fn, fnOpts)) + ) { await optimizeFunction(async () => { await fnOpts?.beforeEach?.("run"); await fn(); @@ -201,7 +204,6 @@ async function runAnalysisTask( return; } - fnOpts?.beforeAll?.("run"); optimizeFunctionSync(() => { fnOpts?.beforeEach?.("run"); fn(); @@ -220,6 +222,23 @@ async function runAnalysisTask( fnOpts?.afterAll?.("run"); } +/** + * Calls `fn` once between its hooks, like tinybench does for functions that + * aren't declared `async`, to tell whether it must be awaited. + */ +async function returnsPromise( + fn: BenchFn, + fnOpts: TinybenchFnOptions | undefined, +): Promise { + await fnOpts?.beforeEach?.("run"); + const result = fn(); + await result; + await fnOpts?.afterEach?.("run"); + return ( + typeof (result as PromiseLike | undefined)?.then === "function" + ); +} + async function runWalltime( registrations: BenchRegistrationInput[], options: BenchRunOptions | undefined, diff --git a/packages/vitest-plugin/tests/fixtures/analysis/promise.bench.ts b/packages/vitest-plugin/tests/fixtures/analysis/promise.bench.ts new file mode 100644 index 00000000..5cfbce81 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/analysis/promise.bench.ts @@ -0,0 +1,24 @@ +import { appendFileSync } from "fs"; +import { test } from "vitest"; + +// Vitest swallows what workers print, so the order of the benchmark's events +// goes to a file. +function log(event: string) { + appendFileSync(process.env.ANALYSIS_LOG_FILE!, `${event}\n`); +} + +test("benchmarks", async ({ bench }) => { + // Returns a promise without being declared `async`. + await bench( + "returns a promise", + { + beforeEach: () => log("beforeEach"), + afterEach: () => log("afterEach"), + }, + () => { + const { promise, resolve } = Promise.withResolvers(); + setTimeout(resolve, 1); + return promise.then(() => log("done")); + }, + ).run(); +}); diff --git a/packages/vitest-plugin/tests/fixtures/analysis/vitest.config.ts b/packages/vitest-plugin/tests/fixtures/analysis/vitest.config.ts new file mode 100644 index 00000000..df313ac4 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/analysis/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; +import codspeedPlugin from "../../../dist/index.mjs"; + +export default defineConfig({ + plugins: [codspeedPlugin()], + define: { + __VERSION__: JSON.stringify("1.0.0"), + }, + test: { + include: ["**/*.bench.ts"], + }, +}); From 16decfc22c03644167f7b26237f56c302385f141 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 9 Nov 2026 15:03:34 +0100 Subject: [PATCH 4/4] fix(tinybench): only call a benchmark in the forks that measure it tinybench calls a function that isn't declared async once in bench.add to tell whether it returns a promise. The walltime root-frame wrapper is sync, so every async benchmark was probed this way, and with forks every process re-registered and probed every benchmark in the file. Benchmarks with side effects ran in forks that never measure them. A process now registers an async no-op for benchmarks it doesn't run. For the benchmarks it runs, an async function is registered with { async: true } so tinybench 6 skips the probe; the wrapper stays sync because an async wrapper adds an await to every iteration. --- packages/tinybench-plugin/src/index.ts | 56 +++++++++++++++---- packages/tinybench-plugin/src/walltime.ts | 17 ++++-- .../tests/fixtures/isolation.mjs | 13 +++-- .../tests/isolation.integ.test.ts | 31 ++++++++-- 4 files changed, 91 insertions(+), 26 deletions(-) diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index bb903ae0..7c8a108e 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -7,14 +7,17 @@ import { SetupInstrumentsRequestBody, SetupInstrumentsResponse, tryIntrospect, + walltimeIsolation, wrapWithRootFrameSync, + type InstrumentMode, + type IsolationMode, type WalltimeOptions, } from "@codspeed/core"; -import { Bench } from "tinybench"; +import { Bench, type FnOptions } from "tinybench"; import { setupCodspeedAnalysisBench } from "./analysis"; import { getOrCreateTaskDataMap } from "./taskData"; import { getOrCreateUriMap } from "./uri"; -import { setupCodspeedWalltimeBench } from "./walltime"; +import { runsTaskFn, setupCodspeedWalltimeBench } from "./walltime"; tryIntrospect(); @@ -29,6 +32,7 @@ export function withCodSpeed( const rootCallingFile = getCallingFile(1); const instrumentMode = getInstrumentMode(); + const isolationMode = getIsolationMode(instrumentMode, options); // Compute and register URI for bench const uriMap = getOrCreateUriMap(bench); @@ -43,25 +47,53 @@ export function withCodSpeed( uri += `::${name}`; uriMap.set(name, uri); taskDataMap.set(name, { fn, fnOpts: opts }); - // In walltime mode the task is driven by tinybench's own measured loop, so - // the root frame must be baked into the function tinybench stores rather - // than injected later (the function is an inaccessible private field on the - // task from tinybench v6 onwards). The sync wrapper is transparent to the - // function's return value, so it preserves tinybench's sync/async handling. - const registeredFn = - instrumentMode === "walltime" ? wrapWithRootFrameSync(fn) : fn; - return rawAdd.bind(bench)(name, registeredFn, opts); + const add = rawAdd.bind(bench); + if (isolationMode === undefined) { + return add(name, fn, opts); + } + // tinybench calls a function that isn't `async` once when it's added, to + // tell whether it returns a promise. A process that doesn't run the task + // registers a stub instead, so the user's function isn't called there. + if (!runsTaskFn(isolationMode, uri)) { + return add(name, notRunHere, opts); + } + // The task is driven by tinybench's own measured loop, so the root frame + // must be baked into the function tinybench stores rather than injected + // later (the function is an inaccessible private field on the task from + // tinybench v6 onwards). An async wrapper would add an await to every + // iteration, so the wrapper stays sync and tinybench is told that an async + // function is async instead (honored from tinybench v6 onwards). + const fnOpts: (FnOptions & { async?: boolean }) | undefined = + fn.constructor.name === "AsyncFunction" ? { ...opts, async: true } : opts; + return add(name, wrapWithRootFrameSync(fn), fnOpts); }; if (instrumentMode === "analysis") { setupCodspeedAnalysisBench(bench, rootCallingFile); - } else if (instrumentMode === "walltime") { - setupCodspeedWalltimeBench(bench, rootCallingFile, options); + } else if (isolationMode !== undefined) { + setupCodspeedWalltimeBench(bench, rootCallingFile, isolationMode); } return bench; } +/** Only walltime runs are isolated. */ +function getIsolationMode( + instrumentMode: InstrumentMode, + options: WalltimeOptions, +): IsolationMode | undefined { + switch (instrumentMode) { + case "walltime": + return walltimeIsolation(options.forks); + case "analysis": + case "disabled": + return undefined; + } +} + +/** Registered for tasks this process never runs; `async` so tinybench doesn't call it. */ +async function notRunHere(): Promise {} + /** * Dynamically setup the CodSpeed instruments. */ diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 7296ef09..95dce247 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -10,13 +10,12 @@ import { runForksSync, samplesToResult, sortedSamplesNs, - walltimeIsolation, writeForkSamples, writeWalltimeResults, type BenchmarkStats, type Benchmark as CodspeedBenchmark, + type IsolationMode, type RoundsResult, - type WalltimeOptions, } from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { getBenchOptions } from "./benchOptions"; @@ -25,14 +24,24 @@ import { BaseBenchRunner } from "./shared"; export function setupCodspeedWalltimeBench( bench: Bench, rootCallingFile: string, - options: WalltimeOptions, + isolationMode: IsolationMode, ): void { - const isolationMode = walltimeIsolation(options.forks); const runner = new WalltimeBenchRunner(bench, rootCallingFile, isolationMode); runner.installInstrumentHooks(); runner.setupBenchMethods(); } +/** + * Whether this process runs the task's function: a fork only runs its own + * task, and the parent of the forks only collects their samples. + */ +export function runsTaskFn(isolationMode: IsolationMode, uri: string): boolean { + return ( + isolationMode.kind === "SharedProcess" || + (isolationMode.kind === "Fork" && isolationMode.uri === uri) + ); +} + class WalltimeBenchRunner extends BaseBenchRunner { private codspeedBenchmarks: CodspeedBenchmark[] = []; diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs index 118c5b07..a4cc9755 100644 --- a/packages/tinybench-plugin/tests/fixtures/isolation.mjs +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -1,5 +1,5 @@ -// Run by `isolation.integ.test.ts`. Prints the PID of every process that runs a -// benchmark. +// Run by `isolation.integ.test.ts`. Prints the PID of every process that calls +// a benchmark's function, and of every process that runs the benchmark. import { Bench } from "tinybench"; import { withCodSpeed } from "../../dist/index.es5.js"; @@ -10,10 +10,15 @@ const bench = withCodSpeed( ); for (const name of ["first", "second"]) { - // tinybench calls the function once when it is added, in every process. let calls = 0; bench.add(name, () => { - if (++calls === 2) { + calls++; + if (calls === 1) { + console.log(`called ${process.pid} ${name}`); + } + // tinybench calls the function once when it is added, to tell whether it + // returns a promise. + if (calls === 2) { console.log(`ran ${process.pid} ${name}`); } }); diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts index f7ec64cd..27e81e6c 100644 --- a/packages/tinybench-plugin/tests/isolation.integ.test.ts +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -18,12 +18,14 @@ const DEFAULT_FORKS = 3; const MIN_ROUNDS_PER_FORK = 4; /** Fails a run whose benchmark process got stuck instead of blocking the suite. */ const RUN_TIMEOUT_MS = 60_000; -const RAN_LINE = /^ran (\d+) (.+)$/; +const PID_LINE = /^(ran|called) (\d+) (.+)$/; interface FixtureRun { status: number | null; /** PIDs that ran each benchmark. */ pidsByName: Map>; + /** PIDs that called each benchmark's function, to run it or not. */ + callerPidsByName: Map>; /** PID of the process the test started. */ parentPid: number; /** Rounds of each benchmark in the results file. */ @@ -80,12 +82,15 @@ async function runFixture({ const [status] = (await once(child, "close")) as [number | null]; const pidsByName = new Map>(); + const callerPidsByName = new Map>(); for (const line of stdout.split("\n")) { - const ran = RAN_LINE.exec(line); - if (!ran) continue; - const pids = pidsByName.get(ran[2]) ?? new Set(); - pids.add(Number(ran[1])); - pidsByName.set(ran[2], pids); + const match = PID_LINE.exec(line); + if (!match) continue; + const [, event, pid, name] = match; + const byName = event === "ran" ? pidsByName : callerPidsByName; + const pids = byName.get(name) ?? new Set(); + pids.add(Number(pid)); + byName.set(name, pids); } const roundsByName = new Map(); @@ -99,6 +104,7 @@ async function runFixture({ return { status, pidsByName, + callerPidsByName, parentPid: child.pid!, roundsByName, }; @@ -153,6 +159,19 @@ describe("walltime forks", () => { RUN_TIMEOUT_MS, ); + it( + "calls a benchmark's function only in the forks that measure it", + async () => { + const run = await runFixture(); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(run.callerPidsByName.get(name)).toEqual(forkPids(run, name)); + } + }, + RUN_TIMEOUT_MS, + ); + it( "runs every benchmark in the parent process when isolation is disabled", async () => {