From 734108ffeb448e7ec84fbc8763f662a3db3751c7 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 9 Nov 2026 19:40:16 +0100 Subject: [PATCH 1/3] feat(core): isolate analysis benchmarks in one fork each Benchmarks measured in one process share allocator, JIT, and heap state, so a benchmark's simulation or memory result depends on the ones that ran before it. Walltime runs already run every benchmark in fresh processes. Add `analysisIsolation()`, which reads the same environment as `walltimeIsolation()` but always uses one fork per benchmark: analysis measures a benchmark once, and `CODSPEED_WALLTIME_FORKS` does not apply. `CODSPEED_PROCESS_ISOLATION=false` keeps every benchmark in one process. Analysis forks reuse `runForks`: the instruments hold the measurement, so a fork writes empty samples, which tell the parent it measured its benchmark. Refs COD-4156 --- packages/core/src/isolation.ts | 28 +++++++++++++-------- packages/core/tests/fixtures/isolation.cjs | 4 +-- packages/core/tests/isolation.integ.test.ts | 21 ++++++++++++---- packages/tinybench-plugin/src/index.ts | 4 +-- packages/vitest-plugin/src/v5/provider.ts | 4 +-- packages/vitest-plugin/src/vitestBackend.ts | 6 ++--- 6 files changed, 43 insertions(+), 24 deletions(-) diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts index e40dd902..6ba97c9b 100644 --- a/packages/core/src/isolation.ts +++ b/packages/core/src/isolation.ts @@ -1,9 +1,10 @@ /** - * Process isolation for walltime runs: every benchmark runs in fresh copies - * of the benchmark process (forks), so the allocator state, JIT state, and - * handles that other benchmarks or setup code leave behind can't change its - * measurement. Rounds within one process are correlated (JIT tier-up, heap - * sizing), so the rounds of several forks are pooled into one result. + * Process isolation: every benchmark runs in fresh copies of the benchmark + * process (forks), so the allocator state, JIT state, and handles that other + * benchmarks or setup code leave behind can't change its measurement. Walltime + * rounds within one process are correlated (JIT tier-up, heap sizing), so the + * rounds of several forks are pooled into one result. Analysis runs measure + * each benchmark once, in one fork. */ import { spawn, spawnSync } from "child_process"; import { randomUUID } from "crypto"; @@ -12,6 +13,7 @@ import { tmpdir } from "os"; import { join } from "path"; import { deserialize, serialize } from "v8"; import { constants, gunzipSync, gzipSync } from "zlib"; +import { getInstrumentMode } from "./runnerMode"; /** Accepts `0`, `false`, `1`, or `true`; defaults to enabled. */ export const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; @@ -44,10 +46,11 @@ export type IsolationMode = | { kind: "Fork"; uri: string; samplesFile: string }; /** - * The isolation of a walltime run, from the environment. `forks` is the - * integration's option, which {@link WALLTIME_FORKS_ENV} overrides. + * The isolation of this run, from the environment. A walltime benchmark runs + * in `forks` forks, the integration's option, which {@link WALLTIME_FORKS_ENV} + * overrides; an analysis benchmark is measured once, so it runs in one. */ -export function walltimeIsolation(forks?: number): IsolationMode { +export function processIsolation(forks?: number): IsolationMode { const uri = process.env[ISOLATED_BENCHMARK_ENV]; if (uri !== undefined) { const samplesFile = process.env[FORK_SAMPLES_FILE_ENV]; @@ -67,7 +70,8 @@ export function walltimeIsolation(forks?: number): IsolationMode { `[CodSpeed] ${PROCESS_ISOLATION_ENV} must be 0, false, 1, or true, got \`${isolation}\``, ); } - return { kind: "Forks", forks: resolveWalltimeForks(forks) }; + const analysis = getInstrumentMode() === "analysis"; + return { kind: "Forks", forks: analysis ? 1 : resolveWalltimeForks(forks) }; } /** @@ -221,7 +225,11 @@ function forkSamples( } } -/** Hands round samples (milliseconds) to the parent as gzip-compressed V8 data. */ +/** + * Hands round samples (milliseconds) to the parent as gzip-compressed V8 data. + * Analysis forks hand none: the instruments hold their measurement, and the + * file only tells the parent that they measured their benchmark. + */ export function writeForkSamples( samplesFile: string, samples: ArrayLike, diff --git a/packages/core/tests/fixtures/isolation.cjs b/packages/core/tests/fixtures/isolation.cjs index bd2d6d2e..29048972 100644 --- a/packages/core/tests/fixtures/isolation.cjs +++ b/packages/core/tests/fixtures/isolation.cjs @@ -1,12 +1,12 @@ // Run by `isolation.integ.test.ts`. Measures benchmark "a" in forks. const { + processIsolation, runForks, runForksSync, - walltimeIsolation, writeForkSamples, } = require("../../dist/index.cjs.js"); -const mode = walltimeIsolation(); +const mode = processIsolation(); function measure() { console.log(`ran ${process.pid} ${JSON.stringify(process.execArgv)}`); diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts index c61d3f45..aaeccb3e 100644 --- a/packages/core/tests/isolation.integ.test.ts +++ b/packages/core/tests/isolation.integ.test.ts @@ -4,9 +4,9 @@ import { FORK_SAMPLES_FILE_ENV, ISOLATED_BENCHMARK_ENV, PROCESS_ISOLATION_ENV, + processIsolation, resolveWalltimeForks, WALLTIME_FORKS_ENV, - walltimeIsolation, } from "../src/isolation"; const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); @@ -21,6 +21,8 @@ const NODE_FLAG = "--no-warnings"; const SAMPLES_PER_FORK = 2; const ENV_KEYS = [ + "CODSPEED_ENV", + "CODSPEED_RUNNER_MODE", PROCESS_ISOLATION_ENV, ISOLATED_BENCHMARK_ENV, FORK_SAMPLES_FILE_ENV, @@ -75,7 +77,7 @@ describe("resolveWalltimeForks", () => { }); }); -describe("walltimeIsolation", () => { +describe("processIsolation", () => { it.each([ [undefined, "Forks"], ["1", "Forks"], @@ -84,12 +86,12 @@ describe("walltimeIsolation", () => { ["false", "SharedProcess"], ])("selects %s as %s", (value, kind) => { setEnv({ [PROCESS_ISOLATION_ENV]: value }); - expect(walltimeIsolation().kind).toBe(kind); + expect(processIsolation().kind).toBe(kind); }); it("runs everything in this process when isolation is off, whatever the fork count", () => { setEnv({ [PROCESS_ISOLATION_ENV]: "false", [WALLTIME_FORKS_ENV]: "4" }); - expect(walltimeIsolation(2).kind).toBe("SharedProcess"); + expect(processIsolation(2).kind).toBe("SharedProcess"); }); it("restricts a fork to its benchmark, whatever the isolation setting", () => { @@ -98,12 +100,21 @@ describe("walltimeIsolation", () => { [ISOLATED_BENCHMARK_ENV]: "file::bench", [FORK_SAMPLES_FILE_ENV]: "samples.json", }); - expect(walltimeIsolation()).toEqual({ + expect(processIsolation()).toEqual({ kind: "Fork", uri: "file::bench", samplesFile: "samples.json", }); }); + + it("runs an analysis benchmark in one fork, whatever the walltime fork count", () => { + setEnv({ + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + [WALLTIME_FORKS_ENV]: "4", + }); + expect(processIsolation(2)).toEqual({ kind: "Forks", forks: 1 }); + }); }); interface FixtureOptions { diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index 7c8a108e..d1518107 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -7,7 +7,7 @@ import { SetupInstrumentsRequestBody, SetupInstrumentsResponse, tryIntrospect, - walltimeIsolation, + processIsolation, wrapWithRootFrameSync, type InstrumentMode, type IsolationMode, @@ -84,7 +84,7 @@ function getIsolationMode( ): IsolationMode | undefined { switch (instrumentMode) { case "walltime": - return walltimeIsolation(options.forks); + return processIsolation(options.forks); case "analysis": case "disabled": return undefined; diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index bc9aeb7c..dc5eea67 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -6,7 +6,7 @@ import { optimizeFunctionSync, setupCore, teardownCore, - walltimeIsolation, + processIsolation, wrapWithRootFrame, wrapWithRootFrameSync, writeForkSamples, @@ -245,7 +245,7 @@ async function runWalltime( getUri: (name: string) => string, test: BenchmarkGroup["test"], ): Promise { - const mode = walltimeIsolation(); + const mode = processIsolation(); if (mode.kind === "Forks") { await queueForks(registrations, options, getUri, test); return registrations.map(({ name }) => analysisResult(name)); diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index d1867bfa..959e0e06 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -1,7 +1,7 @@ import { getGitDir, getInstrumentMode, - walltimeIsolation, + processIsolation, type WalltimeOptions, } from "@codspeed/core"; import { join } from "path"; @@ -60,7 +60,7 @@ class V5Backend implements VitestBackend { configureIsolation(userConfig: ViteUserConfig): void { if (getInstrumentMode() !== "walltime") return; - const isolationMode = walltimeIsolation(); + const isolationMode = processIsolation(); if (isolationMode.kind !== "Fork") return; restrictToBenchmarkFile(userConfig, isolationMode.uri); // The parent's global setup is still up while its forks run: forks skip @@ -78,7 +78,7 @@ class V5Backend implements VitestBackend { options: WalltimeOptions, ): void { if (getInstrumentMode() !== "walltime") return; - const isolationMode = walltimeIsolation(); + const isolationMode = processIsolation(); if (isolationMode.kind === "Fork") { provideParentContext(project); return; From 927aa7c58786c94c01e888d55833d70c183ad02e Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 9 Nov 2026 19:40:19 +0100 Subject: [PATCH 2/3] feat(tinybench): run analysis benchmarks in their own process In simulation and memory mode, the parent now runs each task in one fresh process, which measures only that task. Children keep quiet about the banner and summary, like walltime forks. The in-process integration test mocks the core, which forks cannot see, so it turns isolation off. Refs COD-4156 --- packages/tinybench-plugin/src/analysis.ts | 19 +++++++++-- packages/tinybench-plugin/src/index.ts | 31 +++++------------- packages/tinybench-plugin/src/shared.ts | 26 ++++++++++++--- packages/tinybench-plugin/src/walltime.ts | 18 ++--------- .../tests/index.integ.test.ts | 2 ++ .../tests/isolation.integ.test.ts | 32 +++++++++++++++++-- 6 files changed, 78 insertions(+), 50 deletions(-) diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 6c3fe826..f03d3faf 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -6,6 +6,8 @@ import { optimizeFunctionSync, wrapWithRootFrame, wrapWithRootFrameSync, + writeForkSamples, + type IsolationMode, } from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { BaseBenchRunner } from "./shared"; @@ -13,8 +15,9 @@ import { BaseBenchRunner } from "./shared"; export function setupCodspeedAnalysisBench( bench: Bench, rootCallingFile: string, + isolationMode: IsolationMode, ): void { - const runner = new AnalysisBenchRunner(bench, rootCallingFile); + const runner = new AnalysisBenchRunner(bench, rootCallingFile, isolationMode); runner.setupBenchMethods(); } @@ -28,6 +31,16 @@ class AnalysisBenchRunner extends BaseBenchRunner { return InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; } + private completeTask(uri: string): void { + this.logTaskCompletion(uri, this.taskCompletionMessage()); + if (this.isolationMode.kind === "Fork") { + writeForkSamples(this.isolationMode.samplesFile, []); + } + } + + /** The instruments hold the measurement of the forks. */ + protected collectForks(): void {} + protected async runTaskAsync(task: Task, uri: string): Promise { const { fnOpts, fn } = this.getTaskData(task); @@ -54,7 +67,7 @@ class AnalysisBenchRunner extends BaseBenchRunner { await fnOpts?.afterEach?.call(task, "run"); await fnOpts?.afterAll?.call(task, "run"); - this.logTaskCompletion(uri, this.taskCompletionMessage()); + this.completeTask(uri); } protected runTaskSync(task: Task, uri: string): void { @@ -83,7 +96,7 @@ class AnalysisBenchRunner extends BaseBenchRunner { fnOpts?.afterEach?.call(task, "run"); fnOpts?.afterAll?.call(task, "run"); - this.logTaskCompletion(uri, this.taskCompletionMessage()); + this.completeTask(uri); } protected finalizeAsyncRun(): Task[] { diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index d1518107..cbd9a8f1 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -4,13 +4,11 @@ import { getInstrumentMode, InstrumentHooks, mongoMeasurement, + processIsolation, SetupInstrumentsRequestBody, SetupInstrumentsResponse, tryIntrospect, - processIsolation, wrapWithRootFrameSync, - type InstrumentMode, - type IsolationMode, type WalltimeOptions, } from "@codspeed/core"; import { Bench, type FnOptions } from "tinybench"; @@ -32,7 +30,7 @@ export function withCodSpeed( const rootCallingFile = getCallingFile(1); const instrumentMode = getInstrumentMode(); - const isolationMode = getIsolationMode(instrumentMode, options); + const isolationMode = processIsolation(options.forks); // Compute and register URI for bench const uriMap = getOrCreateUriMap(bench); @@ -48,15 +46,16 @@ export function withCodSpeed( uriMap.set(name, uri); taskDataMap.set(name, { fn, fnOpts: opts }); const add = rawAdd.bind(bench); - if (isolationMode === undefined) { - return add(name, fn, opts); - } // tinybench calls a function that isn't `async` once when it's added, to // tell whether it returns a promise. A process that doesn't run the task // registers a stub instead, so the user's function isn't called there. if (!runsTaskFn(isolationMode, uri)) { return add(name, notRunHere, opts); } + // The analysis runner calls the captured function itself. + if (instrumentMode === "analysis") { + return add(name, fn, opts); + } // The task is driven by tinybench's own measured loop, so the root frame // must be baked into the function tinybench stores rather than injected // later (the function is an inaccessible private field on the task from @@ -69,28 +68,14 @@ export function withCodSpeed( }; if (instrumentMode === "analysis") { - setupCodspeedAnalysisBench(bench, rootCallingFile); - } else if (isolationMode !== undefined) { + setupCodspeedAnalysisBench(bench, rootCallingFile, isolationMode); + } else { setupCodspeedWalltimeBench(bench, rootCallingFile, isolationMode); } return bench; } -/** Only walltime runs are isolated. */ -function getIsolationMode( - instrumentMode: InstrumentMode, - options: WalltimeOptions, -): IsolationMode | undefined { - switch (instrumentMode) { - case "walltime": - return processIsolation(options.forks); - case "analysis": - case "disabled": - return undefined; - } -} - /** Registered for tasks this process never runs; `async` so tinybench doesn't call it. */ async function notRunHere(): Promise {} diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 2d86eced..f97b2569 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,4 +1,10 @@ -import { setupCore, teardownCore, type IsolationMode } from "@codspeed/core"; +import { + runForks, + runForksSync, + setupCore, + teardownCore, + type IsolationMode, +} from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; @@ -12,8 +18,7 @@ export abstract class BaseBenchRunner { constructor( bench: Bench, rootCallingFile: string, - /** Only walltime runs are isolated. */ - protected readonly isolationMode: IsolationMode = { kind: "SharedProcess" }, + protected readonly isolationMode: IsolationMode, ) { this.bench = bench; this.rootCallingFile = rootCallingFile; @@ -72,6 +77,7 @@ export abstract class BaseBenchRunner { protected abstract runTaskSync(task: Task, uri: string): void; protected abstract finalizeAsyncRun(): Task[]; protected abstract finalizeSyncRun(): Task[]; + protected abstract collectForks(task: Task, samples: Float64Array): void; public setupBenchMethods(): void { this.bench.run = async () => { @@ -85,7 +91,12 @@ export abstract class BaseBenchRunner { for (const task of tasks) { const uri = this.getTaskUri(task); - await this.runTaskAsync(task, uri); + if (this.isolationMode.kind === "Forks") { + const samples = await runForks(uri, this.isolationMode.forks); + this.collectForks(task, samples); + } else { + await this.runTaskAsync(task, uri); + } } return this.finalizeAsyncRun(); @@ -100,7 +111,12 @@ export abstract class BaseBenchRunner { for (const task of tasks) { const uri = this.getTaskUri(task); - this.runTaskSync(task, uri); + if (this.isolationMode.kind === "Forks") { + const samples = runForksSync(uri, this.isolationMode.forks); + this.collectForks(task, samples); + } else { + this.runTaskSync(task, uri); + } } return this.finalizeSyncRun(); diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 95dce247..6c3bf483 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -6,8 +6,6 @@ import { mongoMeasurement, msToNs, msToS, - runForks, - runForksSync, samplesToResult, sortedSamplesNs, writeForkSamples, @@ -110,12 +108,6 @@ class WalltimeBenchRunner extends BaseBenchRunner { } protected async runTaskAsync(task: Task, uri: string): Promise { - const { isolationMode } = this; - if (isolationMode.kind === "Forks") { - const samples = await runForks(uri, isolationMode.forks); - return this.registerForkedBenchmark(task, samples); - } - // run the warmup of the task right before its actual run if (getBenchOptions(this.bench).warmup) { await task.warmup(); @@ -128,13 +120,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { this.registerCodspeedBenchmarkFromTask(task); } - protected runTaskSync(task: Task, uri: string): void { - const { isolationMode } = this; - if (isolationMode.kind === "Forks") { - const samples = runForksSync(uri, isolationMode.forks); - return this.registerForkedBenchmark(task, samples); - } - + protected runTaskSync(task: Task): void { if (getBenchOptions(this.bench).warmup) { task.warmupSync(); } @@ -168,7 +154,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { this.registerCodspeedBenchmark(task, task.result); } - private registerForkedBenchmark(task: Task, samples: Float64Array): void { + protected collectForks(task: Task, samples: Float64Array): void { if (samples.length === 0) { return this.warnNoResult(task); } diff --git a/packages/tinybench-plugin/tests/index.integ.test.ts b/packages/tinybench-plugin/tests/index.integ.test.ts index 3b0ec727..2b85897f 100644 --- a/packages/tinybench-plugin/tests/index.integ.test.ts +++ b/packages/tinybench-plugin/tests/index.integ.test.ts @@ -44,6 +44,8 @@ vi.mock("@codspeed/core", async (importOriginal) => { beforeEach(() => { process.env.CODSPEED_ENV = "true"; process.env.CODSPEED_RUNNER_MODE = "instrumentation"; + // The mocked core only exists in this process, forks would not see it. + process.env.CODSPEED_PROCESS_ISOLATION = "false"; vi.clearAllMocks(); }); diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts index 27e81e6c..c080a7a4 100644 --- a/packages/tinybench-plugin/tests/isolation.integ.test.ts +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -1,7 +1,7 @@ import { getV8Flags } from "@codspeed/core"; import { spawn } from "child_process"; import { once } from "events"; -import { mkdtempSync, readdirSync, readFileSync, rmSync } from "fs"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; import { fileURLToPath } from "url"; @@ -40,6 +40,7 @@ interface FixtureOptions { /** `CODSPEED_WALLTIME_FORKS`. */ forksEnv?: number; sync?: boolean; + mode?: "walltime" | "simulation"; } async function runFixture({ @@ -47,13 +48,14 @@ async function runFixture({ forksOption, forksEnv, sync, + mode = "walltime", }: FixtureOptions = {}): Promise { const profileFolder = mkdtempSync(join(tmpdir(), "codspeed-tinybench-")); try { const env: NodeJS.ProcessEnv = { ...process.env, CODSPEED_ENV: "true", - CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_RUNNER_MODE: mode, CODSPEED_PROFILE_FOLDER: profileFolder, }; delete env[PROCESS_ISOLATION_ENV]; @@ -95,7 +97,9 @@ async function runFixture({ const roundsByName = new Map(); const resultsDir = join(profileFolder, "results"); - for (const file of readdirSync(resultsDir)) { + // Only walltime runs write results: the instruments hold analysis ones. + const resultFiles = existsSync(resultsDir) ? readdirSync(resultsDir) : []; + for (const file of resultFiles) { const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")); for (const benchmark of data.benchmarks) { roundsByName.set(benchmark.name, benchmark.stats.rounds); @@ -185,3 +189,25 @@ describe("walltime forks", () => { RUN_TIMEOUT_MS, ); }); + +describe("simulation isolation", () => { + it.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, + ])( + "runs every benchmark in its own process ($name run)", + async ({ sync }) => { + const run = await runFixture({ sync, mode: "simulation" }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(1); + } + const all = new Set( + BENCHMARKS.flatMap((name) => [...forkPids(run, name)]), + ); + expect(all.size).toBe(BENCHMARKS.length); + }, + RUN_TIMEOUT_MS, + ); +}); From 7d16c571810f2818ff41949ee79663426c90c20a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 9 Nov 2026 19:40:23 +0100 Subject: [PATCH 3/3] feat(vitest): run analysis benchmarks in their own process With Vitest 5, simulation and memory runs now reuse the walltime fork flow: workers queue each benchmark instead of measuring it, and the main process runs one fresh `vitest bench` per benchmark once the run is over. The fork is restricted to the benchmark's file and measures only that benchmark. `CODSPEED_WALLTIME_FORKS` and the `forks` options do not apply: analysis measures a benchmark once. Vitest 3/4 still don't fork. Closes COD-4156 --- .../src/__tests__/isolation.test.ts | 25 +++++++++++++++-- packages/vitest-plugin/src/v5/forkReporter.ts | 9 +++++- packages/vitest-plugin/src/v5/provider.ts | 28 ++++++++++++------- packages/vitest-plugin/src/vitestBackend.ts | 7 ++--- 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts index dae589b5..43be6d84 100644 --- a/packages/vitest-plugin/src/__tests__/isolation.test.ts +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -1,6 +1,7 @@ import { spawn } from "child_process"; import { once } from "events"; import { + existsSync, mkdtempSync, readdirSync, readFileSync, @@ -44,6 +45,7 @@ interface FixtureOptions { pluginForks?: number; firstForks?: number; failing?: boolean; + mode?: "walltime" | "simulation"; } async function runFixture(options: FixtureOptions = {}) { @@ -53,7 +55,7 @@ async function runFixture(options: FixtureOptions = {}) { const env: NodeJS.ProcessEnv = { ...process.env, CODSPEED_ENV: "true", - CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_RUNNER_MODE: options.mode ?? "walltime", CODSPEED_PROFILE_FOLDER: profileFolder, ISOLATION_LOG_FILE: logFile, [PROCESS_ISOLATION_ENV]: options.isolation, @@ -85,7 +87,9 @@ async function runFixture(options: FixtureOptions = {}) { const roundsByName = new Map(); const resultsDir = join(profileFolder, "results"); - for (const file of status === 0 ? readdirSync(resultsDir) : []) { + // Only walltime runs write results: the instruments hold analysis ones. + const hasResults = status === 0 && existsSync(resultsDir); + for (const file of hasResults ? readdirSync(resultsDir) : []) { const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")); for (const benchmark of data.benchmarks) { roundsByName.set(benchmark.name, benchmark.stats.rounds); @@ -147,3 +151,20 @@ describe("walltime forks", () => { RUN_TIMEOUT_MS, ); }); + +describe("simulation isolation", () => { + it( + "runs every benchmark in its own process", + async () => { + const run = await runFixture({ mode: "simulation", forksEnv: 2 }); + + expect(run.status, run.stderr).toBe(0); + const pids = ["first", "second", "third"].map((name) => { + expect(run.pidsByName.get(name)?.size).toBe(1); + return [...run.pidsByName.get(name)!][0]; + }); + expect(new Set(pids).size).toBe(pids.length); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/src/v5/forkReporter.ts b/packages/vitest-plugin/src/v5/forkReporter.ts index 7055eb7b..3009d154 100644 --- a/packages/vitest-plugin/src/v5/forkReporter.ts +++ b/packages/vitest-plugin/src/v5/forkReporter.ts @@ -1,4 +1,5 @@ import { + getInstrumentMode, resolveWalltimeForks, runForks, samplesToResult, @@ -15,7 +16,8 @@ import { withForkContext } from "./forkContext"; /** * Measures the benchmarks the workers queued (see `provider.ts`) in fresh - * processes and reports their pooled rounds. Runs in the main process once the + * processes and reports their pooled walltime rounds; analysis forks report to + * the instruments themselves. Runs in the main process once the * run is over: every file has torn down, so resources its hooks held (ports, * databases, ...) are free for the forks. The global setup is still up (it * tears down on close), so the forks skip it and inherit what it provided. @@ -40,6 +42,11 @@ async function measureInForks( for (const module of modules) { for (const test of module.children.allTests()) { for (const queued of forkedBenchmarks(test.meta())) { + if (getInstrumentMode() !== "walltime") { + // Analysis measures a benchmark once, so it needs only one fork. + await runForks(queued.uri, 1); + continue; + } const forks = resolveWalltimeForks(queued.forks ?? options.forks); const samples = await runForks(queued.uri, forks); const benchmark = tinybenchTaskToBenchmark( diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index dc5eea67..ee4a6d76 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -4,13 +4,14 @@ import { InstrumentHooks, optimizeFunction, optimizeFunctionSync, + processIsolation, setupCore, teardownCore, - processIsolation, wrapWithRootFrame, wrapWithRootFrameSync, writeForkSamples, type Benchmark, + type IsolationMode, type WalltimeOptions, } from "@codspeed/core"; import { createRequire } from "module"; @@ -149,24 +150,35 @@ const provider: BenchmarkProvider = { const groupUri = buildGroupUri(test); const getUri = (name: string) => `${groupUri}::${name}`; + const mode = processIsolation(); + if (mode.kind === "Forks") { + await queueForks(registrations, options, getUri, test); + return registrations.map(({ name }) => analysisResult(name)); + } if (isWalltime) { - return runWalltime(registrations, options, getUri, test); + return runWalltime(registrations, options, getUri, test, mode); } - return runAnalysis(registrations, getUri); + return runAnalysis(registrations, getUri, mode); }, }; async function runAnalysis( registrations: BenchRegistrationInput[], getUri: (name: string) => string, + mode: IsolationMode, ): Promise { const label = InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; const results: BenchResult[] = []; for (const { name, fn, fnOpts } of registrations) { const uri = getUri(name); + results.push(analysisResult(name)); + // A fork measures only its benchmark; Vitest still gets every result. + if (mode.kind === "Fork" && mode.uri !== uri) continue; await runAnalysisTask(fn, fnOpts, uri); console.log(`[CodSpeed] ${label} ${uri}`); - results.push(analysisResult(name)); + if (mode.kind === "Fork") { + writeForkSamples(mode.samplesFile, []); + } } return results; } @@ -244,12 +256,8 @@ async function runWalltime( options: BenchRunOptions | undefined, getUri: (name: string) => string, test: BenchmarkGroup["test"], + mode: IsolationMode, ): Promise { - const mode = processIsolation(); - if (mode.kind === "Forks") { - await queueForks(registrations, options, getUri, test); - return registrations.map(({ name }) => analysisResult(name)); - } // A fork measures only its benchmark; Vitest still gets every result. const measured = registrations.filter( ({ name }) => mode.kind !== "Fork" || mode.uri === getUri(name), @@ -296,7 +304,7 @@ async function runWalltime( /** * Queues every benchmark for measurement in fresh processes once the run is - * over (see `forkReporter.ts`). Vitest gets zeroed results, like in analysis mode. + * over (see `forkReporter.ts`). Vitest gets zeroed results. */ async function queueForks( registrations: BenchRegistrationInput[], diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index 959e0e06..cab84f2f 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -58,8 +58,6 @@ class V5Backend implements VitestBackend { } configureIsolation(userConfig: ViteUserConfig): void { - if (getInstrumentMode() !== "walltime") return; - const isolationMode = processIsolation(); if (isolationMode.kind !== "Fork") return; restrictToBenchmarkFile(userConfig, isolationMode.uri); @@ -69,15 +67,14 @@ class V5Backend implements VitestBackend { } /** - * The parent of a forked walltime run measures nothing in its workers: they - * queue the benchmarks, and the main process forks them once the run is over. + * The parent of a forked run measures nothing in its workers: they queue the + * benchmarks, and the main process forks them once the run is over. */ configureVitest( project: TestProject, vitest: Vitest, options: WalltimeOptions, ): void { - if (getInstrumentMode() !== "walltime") return; const isolationMode = processIsolation(); if (isolationMode.kind === "Fork") { provideParentContext(project);