From 8c6b103b7e413384f3461ddd75be5fe0bef2d7c6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:35 +0200 Subject: [PATCH 1/6] feat(core): add per-benchmark process isolation Benchmarks of one process share allocator, JIT and thread state, so what one benchmark leaves behind (raised mmap threshold, fragmented heap, optimized code, retained caches) changes the measurement of every benchmark after it. Adding or changing one benchmark can then shift unrelated results. Add `Isolation` for the integrations. It is on by default and turned off with `CODSPEED_PROCESS_ISOLATION=false`. `Isolation.run` re-executes the current command with `CODSPEED_ISOLATED_BENCHMARK` set to one benchmark URI, so each benchmark gets a fresh process. The child only runs that benchmark, which overrides user filters. - `CODSPEED_PROCESS_ISOLATION`: unset, `1` or `true` is on, `0` or `false` is off, anything else is an error. - `CODSPEED_ISOLATED_BENCHMARK`: set by the parent on each child. - `CODSPEED_ISOLATION_COMMAND`: JSON command to re-run, for runners that measure in worker processes whose argv is not the benchmark command. The child is started with `process.execPath` and `process.execArgv`, so the V8 flags set by the runner survive. A failing child ends the parent with the same exit code (128 + N when signal N killed it). Children run sequentially with `spawnSync`, which keeps the output in order. Re-exec is used instead of fork because fork only copies the calling thread. --- packages/core/src/index.ts | 1 + packages/core/src/isolation.ts | 182 +++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 packages/core/src/isolation.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 63ac394f..514abe22 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -51,6 +51,7 @@ export type { SetupInstrumentsResponse, } from "./generated/openapi"; export { getV8Flags, tryIntrospect } from "./introspection"; +export * from "./isolation"; export { getUnsupportedNodeVersionWarning, SUPPORTED_NODE_MAJORS, diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts new file mode 100644 index 00000000..0a8e013a --- /dev/null +++ b/packages/core/src/isolation.ts @@ -0,0 +1,182 @@ +/** + * Process isolation: every benchmark runs in a fresh copy of the benchmark + * process, so the allocator state, JIT state, and handles that other + * benchmarks or setup code leave behind can't change its measurement. + */ +import { spawnSync, type SpawnSyncReturns } from "child_process"; +import { constants } from "os"; + +/** Isolation is on unless this is `0` or `false`. */ +export const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +/** Set on each child process: the URI of the only benchmark it runs. */ +export const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; +/** + * JSON array `[executable, ...args]` of the command that runs the benchmarks + * again. Integrations whose benchmarks run in a worker process (vitest) set it + * from the main process, because re-executing the worker alone can't work. + */ +export const ISOLATION_COMMAND_ENV = "CODSPEED_ISOLATION_COMMAND"; + +/** Exit code for a child that ended with neither an exit code nor a signal. */ +const GENERIC_FAILURE_EXIT_CODE = 1; +/** Shells report a child killed by signal N as exit code 128 + N. */ +const SIGNAL_EXIT_CODE_OFFSET = 128; + +/** How an integration runs each benchmark. */ +export type IsolationMode = + /** All benchmarks share this process and its state. */ + | { kind: "SharedProcess" } + /** Every benchmark runs in its own child process. */ + | { kind: "ProcessPerBenchmark" } + /** A child spawned in `ProcessPerBenchmark` mode, restricted to `uri`. */ + | { kind: "IsolatedBenchmark"; uri: string }; + +export class Isolation { + private static instance: Isolation | undefined; + private static readonly sharedInstance = new Isolation({ + kind: "SharedProcess", + }); + + private constructor(readonly mode: IsolationMode) {} + + /** + * The isolation of runs where no instrument measures anything (the hooks are + * not active), so there is nothing to isolate. + */ + static shared(): Isolation { + return Isolation.sharedInstance; + } + + /** The isolation of this process, read once from the environment. */ + static current(): Isolation { + Isolation.instance ??= new Isolation(modeFromEnv()); + return Isolation.instance; + } + + /** The only benchmark this process may run. It overrides user filters. */ + onlyBenchmark(): string | undefined { + switch (this.mode.kind) { + case "IsolatedBenchmark": + return this.mode.uri; + case "SharedProcess": + case "ProcessPerBenchmark": + return undefined; + } + } + + /** Whether this process must run the benchmark `uri`. */ + shouldRun(uri: string): boolean { + const only = this.onlyBenchmark(); + return only === undefined || only === uri; + } + + /** Whether this process is a child that runs one benchmark for a parent. */ + isChild(): boolean { + return this.onlyBenchmark() !== undefined; + } + + /** + * Runs the benchmark `uri`. `bench` must contain the whole measurement, + * from `startBenchmark` to `setExecutedBenchmark`. + */ + async run(uri: string, bench: () => Promise | void): Promise { + switch (this.mode.kind) { + case "SharedProcess": + case "IsolatedBenchmark": + await bench(); + return; + case "ProcessPerBenchmark": + runInChildProcess(uri); + return; + } + } + + /** Synchronous variant of {@link Isolation.run}. */ + runSync(uri: string, bench: () => void): void { + switch (this.mode.kind) { + case "SharedProcess": + case "IsolatedBenchmark": + bench(); + return; + case "ProcessPerBenchmark": + runInChildProcess(uri); + return; + } + } +} + +function modeFromEnv(): IsolationMode { + const uri = process.env[ISOLATED_BENCHMARK_ENV]; + if (uri !== undefined) { + return { kind: "IsolatedBenchmark", uri }; + } + + const value = process.env[PROCESS_ISOLATION_ENV]; + switch (value) { + case undefined: + case "1": + case "true": + return { kind: "ProcessPerBenchmark" }; + case "0": + case "false": + return { kind: "SharedProcess" }; + default: + throw new Error( + `${PROCESS_ISOLATION_ENV} must be \`true\` or \`false\`, got \`${value}\``, + ); + } +} + +/** + * Records the command of this process as the one that children run. Call it + * from the main process of integrations that measure in worker processes. It + * overwrites a command inherited from an outer process, which is not ours. + */ +export function recordIsolationCommand(): void { + process.env[ISOLATION_COMMAND_ENV] = JSON.stringify(currentCommand()); +} + +function currentCommand(): string[] { + // execArgv keeps the V8 flags (e.g. --perf-prof) that the runner set. + return [process.execPath, ...process.execArgv, ...process.argv.slice(1)]; +} + +function isolationCommand(): string[] { + const recorded = process.env[ISOLATION_COMMAND_ENV]; + if (recorded === undefined) { + return currentCommand(); + } + return JSON.parse(recorded) as string[]; +} + +/** + * Runs the benchmark command again, limited to `uri`, and exits with the + * child's code if it fails. Re-exec, not fork(): Node can't fork, and a fork + * would only copy the calling thread. + */ +function runInChildProcess(uri: string): void { + const [executable, ...args] = isolationCommand(); + // spawnSync blocks the event loop, so stdout written before this call is + // already flushed and output stays in order. + const result = spawnSync(executable, args, { + stdio: "inherit", + env: { ...process.env, [ISOLATED_BENCHMARK_ENV]: uri }, + }); + if (result.error !== undefined) { + throw new Error( + `[CodSpeed] failed to start the process for ${uri}: ${result.error.message}`, + ); + } + if (result.status === 0) { + return; + } + process.exit(exitCode(result)); +} + +/** Exit code of a failed child, in the shell convention. */ +function exitCode(result: SpawnSyncReturns): number { + if (result.signal !== null) { + return SIGNAL_EXIT_CODE_OFFSET + constants.signals[result.signal]; + } + return result.status ?? GENERIC_FAILURE_EXIT_CODE; +} From 7b9f42adabe9c9027f526f4afb6a3bd8c1a98269 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:36 +0200 Subject: [PATCH 2/6] feat(benchmark.js): run each benchmark in its own process Route every instrumented benchmark through `Isolation.run`. A child process only runs its own benchmark and skips the banner, the summary line and the completion listeners that the parent already handles. A run call that owns no benchmark of the child does not touch the instruments. --- packages/benchmark.js-plugin/src/index.ts | 130 +++++++++++++--------- 1 file changed, 80 insertions(+), 50 deletions(-) diff --git a/packages/benchmark.js-plugin/src/index.ts b/packages/benchmark.js-plugin/src/index.ts index fa69a35a..747b5fc2 100644 --- a/packages/benchmark.js-plugin/src/index.ts +++ b/packages/benchmark.js-plugin/src/index.ts @@ -1,6 +1,7 @@ import { getCallingFile, InstrumentHooks, + Isolation, mongoMeasurement, optimizeFunction, optimizeFunctionSync, @@ -168,63 +169,92 @@ async function runBenchmarks({ baseUri, benchmarkCompletedListeners, }: RunBenchmarksOptions): Promise { - console.log(`[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`); + const isolation = Isolation.current(); + const isChild = isolation.isChild(); + const selected = benches + .map((bench, i) => ({ + bench, + uri: bench.uri ?? `${baseUri}::unknown_${i}`, + })) + .filter(({ uri }) => isolation.shouldRun(uri)); + // A child only measures one benchmark. A run call that owns none of them + // must not touch the instruments, the call that owns it does. + if (isChild && selected.length === 0) { + return; + } + + // The parent already printed the banner and the summary, children stay quiet. + if (!isChild) { + console.log( + `[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`, + ); + } setupCore(); - for (let i = 0; i < benches.length; i++) { - const bench = benches[i]; - const uri = bench.uri ?? `${baseUri}::unknown_${i}`; - const isAsync = bench.options.async || bench.options.defer; - let benchPayload; - if (bench.options.defer) { - benchPayload = () => { - return new Promise((resolve, reject) => { - (bench.fn as CallableFunction)({ resolve, reject }); - }); - }; - } else if (bench.options.async) { - benchPayload = async () => { - await (bench.fn as CallableFunction)(); - }; - } else { - benchPayload = bench.fn as CallableFunction; + for (const { bench, uri } of selected) { + await isolation.run(uri, () => measureBenchmark(bench, uri)); + // The parent reports for every benchmark, the child ran only one of them. + if (!isChild) { + benchmarkCompletedListeners.forEach((listener) => listener()); } + } + teardownCore(); + if (!isChild) { + console.log(`[CodSpeed] Done running ${benches.length} benches.`); + } +} - if (typeof bench.options.setup === "function") { - await bench.options.setup(); - } +async function measureBenchmark( + bench: BenchmarkWithOptions, + uri: string, +): Promise { + const isAsync = bench.options.async || bench.options.defer; + let benchPayload; + if (bench.options.defer) { + benchPayload = () => { + const { promise, resolve, reject } = Promise.withResolvers(); + (bench.fn as CallableFunction)({ resolve, reject }); + return promise; + }; + } else if (bench.options.async) { + benchPayload = async () => { + await (bench.fn as CallableFunction)(); + }; + } else { + benchPayload = bench.fn as CallableFunction; + } - if (isAsync) { - await optimizeFunction(benchPayload); - await mongoMeasurement.start(uri); - global.gc?.(); - await wrapWithRootFrame(async () => { - InstrumentHooks.startBenchmark(); - await benchPayload(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - } else { - optimizeFunctionSync(benchPayload); - await mongoMeasurement.start(uri); - wrapWithRootFrameSync(() => { - InstrumentHooks.startBenchmark(); - benchPayload(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - } + if (typeof bench.options.setup === "function") { + await bench.options.setup(); + } - if (typeof bench.options.teardown === "function") { - await bench.options.teardown(); - } + if (isAsync) { + await optimizeFunction(benchPayload); + await mongoMeasurement.start(uri); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await benchPayload(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await mongoMeasurement.stop(uri); + } else { + optimizeFunctionSync(benchPayload); + await mongoMeasurement.start(uri); + wrapWithRootFrameSync(() => { + InstrumentHooks.startBenchmark(); + benchPayload(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await mongoMeasurement.stop(uri); + } - console.log(` ✔ Measured ${uri}`); - benchmarkCompletedListeners.forEach((listener) => listener()); + if (typeof bench.options.teardown === "function") { + await bench.options.teardown(); } - teardownCore(); - console.log(`[CodSpeed] Done running ${benches.length} benches.`); + + console.log(` ✔ Measured ${uri}`); } /** From cf218bdae141937d9da01abb2e91be097b304025 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:37 +0200 Subject: [PATCH 3/6] feat(tinybench): run each benchmark in its own process The analysis runner measures every task through `Isolation.run` (async and sync). Walltime runs and runs without active instrument hooks are not isolated. A child only measures its own task and stays quiet about the banner and the summary line. --- packages/tinybench-plugin/src/analysis.ts | 7 ++- packages/tinybench-plugin/src/shared.ts | 53 ++++++++++++++++++----- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 65bd6183..750925e5 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -1,6 +1,7 @@ import { getCodspeedRunnerMode, InstrumentHooks, + Isolation, mongoMeasurement, optimizeFunction, optimizeFunctionSync, @@ -14,7 +15,11 @@ export function setupCodspeedAnalysisBench( bench: Bench, rootCallingFile: string, ): void { - const runner = new AnalysisBenchRunner(bench, rootCallingFile); + // Without instrumentation nothing is measured, so there is nothing to isolate. + const isolation = InstrumentHooks.isInstrumented() + ? Isolation.current() + : Isolation.shared(); + const runner = new AnalysisBenchRunner(bench, rootCallingFile, isolation); runner.setupBenchMethods(); } diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 0fb811c5..14828bac 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,4 +1,4 @@ -import { setupCore, teardownCore } from "@codspeed/core"; +import { Isolation, setupCore, teardownCore } from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; @@ -8,10 +8,19 @@ declare const __VERSION__: string; export abstract class BaseBenchRunner { protected bench: Bench; protected rootCallingFile: string; - - constructor(bench: Bench, rootCallingFile: string) { + /** Runners that measure pass the isolation of this process; walltime shares it. */ + private readonly isolation: Isolation; + private readonly isChild: boolean; + + constructor( + bench: Bench, + rootCallingFile: string, + isolation: Isolation = Isolation.shared(), + ) { this.bench = bench; this.rootCallingFile = rootCallingFile; + this.isolation = isolation; + this.isChild = isolation.isChild(); } private setupBenchRun(): void { @@ -20,6 +29,10 @@ export abstract class BaseBenchRunner { } private logStart(): void { + // The parent already printed the banner, children stay quiet. + if (this.isChild) { + return; + } console.log( `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (${this.getModeName()})`, ); @@ -45,7 +58,12 @@ export abstract class BaseBenchRunner { protected finalizeBenchRun(): Task[] { teardownCore(); - console.log(`[CodSpeed] Done running ${this.bench.tasks.length} benches.`); + // The parent already printed the summary line, children stay quiet. + if (!this.isChild) { + console.log( + `[CodSpeed] Done running ${this.bench.tasks.length} benches.`, + ); + } return this.bench.tasks; } @@ -57,25 +75,40 @@ export abstract class BaseBenchRunner { public setupBenchMethods(): void { this.bench.run = async () => { + const tasks = this.selectTasks(); + // A child that owns none of this bench's tasks must not touch the + // instruments, the bench that owns its task does. + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); - await this.runTaskAsync(task, uri); + for (const { task, uri } of tasks) { + await this.isolation.run(uri, () => this.runTaskAsync(task, uri)); } return this.finalizeAsyncRun(); }; this.bench.runSync = () => { + const tasks = this.selectTasks(); + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); - this.runTaskSync(task, uri); + for (const { task, uri } of tasks) { + this.isolation.runSync(uri, () => this.runTaskSync(task, uri)); } return this.finalizeSyncRun(); }; } + + /** The tasks this process measures: all of them, or the isolated one. */ + private selectTasks(): { task: Task; uri: string }[] { + return this.bench.tasks + .map((task) => ({ task, uri: this.getTaskUri(task) })) + .filter(({ uri }) => this.isolation.shouldRun(uri)); + } } From b14741529de5fbeef4fc36a5a7d72a8fe1e628c4 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:37 +0200 Subject: [PATCH 4/6] feat(vitest): run each benchmark in its own process Both the Vitest 5 provider and the legacy analysis runner measure through `Isolation.run`. An isolated child still reports every benchmark to Vitest, but only measures its own. Vitest measures in forked workers, and the argv of a worker is not the command that starts the benchmarks. The plugin records the command of the main Vitest process in `CODSPEED_ISOLATION_COMMAND` from its `config()` hook, before the workers are forked, and each worker re-runs that command once per benchmark. This means one full Vitest run per benchmark. --- packages/vitest-plugin/src/index.ts | 5 + packages/vitest-plugin/src/legacy/analysis.ts | 99 ++++++++++++------- packages/vitest-plugin/src/v5/provider.ts | 16 ++- 3 files changed, 84 insertions(+), 36 deletions(-) diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index cb3af1ae..ae81cf3b 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -3,6 +3,7 @@ import { getV8Flags, InstrumentHooks, mongoMeasurement, + recordIsolationCommand, SetupInstrumentsRequestBody, SetupInstrumentsResponse, } from "@codspeed/core"; @@ -38,6 +39,10 @@ export default function codspeedPlugin(): Plugin { }, enforce: "post", config(userConfig): ViteUserConfig { + // Vitest measures in forked workers, whose argv is not the command that + // starts the benchmarks. Record the command of this main process so the + // workers can run it again for each benchmark. Workers inherit the env. + recordIsolationCommand(); return { test: { pool: "forks", diff --git a/packages/vitest-plugin/src/legacy/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts index f71f0c0a..7dc9f7d3 100644 --- a/packages/vitest-plugin/src/legacy/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -1,5 +1,6 @@ import { InstrumentHooks, + Isolation, logDebug, mongoMeasurement, optimizeFunction, @@ -41,44 +42,50 @@ async function runAnalysisBench( suite: RunnerTestSuite, currentSuiteName: string, tinybenchModule: Tinybench, + isolation: Isolation, ) { const uri = `${currentSuiteName}::${benchmark.name}`; - const fn = getBenchFn(benchmark); - - // Constructing a Bench applies tinybench's no-op defaults for the setup and - // teardown hooks and gives them the Task they expect. The bench itself is - // never run: this runner drives the benchmark function directly. - const bench = new tinybenchModule.Bench(getBenchOptions(benchmark)); - const task = new tinybenchModule.Task(bench, benchmark.name, fn); - - await bench.setup(task, "warmup"); - await optimizeFunction(async () => { + if (!isolation.shouldRun(uri)) return; + + await isolation.run(uri, async () => { + const fn = getBenchFn(benchmark); + + // Constructing a Bench applies tinybench's no-op defaults for the setup and + // teardown hooks and gives them the Task they expect. The bench itself is + // never run: this runner drives the benchmark function directly. + const bench = new tinybenchModule.Bench(getBenchOptions(benchmark)); + const task = new tinybenchModule.Task(bench, benchmark.name, fn); + + await bench.setup(task, "warmup"); + await optimizeFunction(async () => { + await callSuiteHook(suite, benchmark, "beforeEach"); + await fn(); + await callSuiteHook(suite, benchmark, "afterEach"); + }); + await bench.teardown(task, "warmup"); + + await bench.setup(task, "run"); await callSuiteHook(suite, benchmark, "beforeEach"); - await fn(); + await mongoMeasurement.start(uri); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await mongoMeasurement.stop(uri); await callSuiteHook(suite, benchmark, "afterEach"); + await bench.teardown(task, "run"); + + logCodSpeed(`${uri} done`); }); - await bench.teardown(task, "warmup"); - - await bench.setup(task, "run"); - await callSuiteHook(suite, benchmark, "beforeEach"); - await mongoMeasurement.start(uri); - global.gc?.(); - await wrapWithRootFrame(async () => { - InstrumentHooks.startBenchmark(); - await fn(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - await callSuiteHook(suite, benchmark, "afterEach"); - await bench.teardown(task, "run"); - - logCodSpeed(`${uri} done`); } async function runAnalysisBenchmarkSuite( suite: RunnerTestSuite, tinybenchModule: Tinybench, + isolation: Isolation, parentSuiteName?: string, ) { const currentSuiteName = parentSuiteName @@ -91,9 +98,20 @@ async function runAnalysisBenchmarkSuite( if (task.mode !== "run") continue; if (task.type === "suite") { - await runAnalysisBenchmarkSuite(task, tinybenchModule, currentSuiteName); + await runAnalysisBenchmarkSuite( + task, + tinybenchModule, + isolation, + currentSuiteName, + ); } else if (isVitestTaskBenchmark(task)) { - await runAnalysisBench(task, suite, currentSuiteName, tinybenchModule); + await runAnalysisBench( + task, + suite, + currentSuiteName, + tinybenchModule, + isolation, + ); } } @@ -107,9 +125,24 @@ export class AnalysisRunner extends NodeBenchmarkRunner { patchRootSuiteWithFullFilePath(suite); - logCodSpeed(`running suite ${suite.name}`); - await runAnalysisBenchmarkSuite(suite, await this.importTinybench()); - logCodSpeed(`running suite ${suite.name} done`); + // Without instrumentation nothing is measured, so there is nothing to isolate. + const isolation = InstrumentHooks.isInstrumented() + ? Isolation.current() + : Isolation.shared(); + // The parent already printed the suite lines, children stay quiet. + const isChild = isolation.isChild(); + + if (!isChild) { + logCodSpeed(`running suite ${suite.name}`); + } + await runAnalysisBenchmarkSuite( + suite, + await this.importTinybench(), + isolation, + ); + if (!isChild) { + logCodSpeed(`running suite ${suite.name} done`); + } teardownCore(); } diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index ef38fe0e..c54d2017 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -2,6 +2,7 @@ import { getGitDir, getInstrumentMode, InstrumentHooks, + Isolation, optimizeFunction, optimizeFunctionSync, setupCore, @@ -155,12 +156,21 @@ async function runAnalysis( registrations: BenchRegistrationInput[], getUri: (name: string) => string, ): Promise { - const label = InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; + const instrumented = InstrumentHooks.isInstrumented(); + const label = instrumented ? "Measured" : "Checked"; + // Without instrumentation nothing is measured, so there is nothing to isolate. + const isolation = instrumented ? Isolation.current() : Isolation.shared(); const results: BenchResult[] = []; for (const { name, fn, fnOpts } of registrations) { const uri = getUri(name); - await runAnalysisTask(fn, fnOpts, uri); - console.log(`[CodSpeed] ${label} ${uri}`); + // An isolated child still reports every benchmark to Vitest, but it only + // measures its own. + if (isolation.shouldRun(uri)) { + await isolation.run(uri, async () => { + await runAnalysisTask(fn, fnOpts, uri); + console.log(`[CodSpeed] ${label} ${uri}`); + }); + } results.push(analysisResult(name)); } return results; From d4c8e425662a3d17faec7a23c6ea885159d89e00 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:45 +0200 Subject: [PATCH 5/6] test: cover process isolation end to end Run fixtures in simulation mode with the instrument hooks stubbed (they only exist under the CodSpeed runner) and check, for core, benchmark.js, tinybench and Vitest 5, that: - every benchmark runs in a fresh process by default; - `CODSPEED_PROCESS_ISOLATION=false` runs them all in one process; - a failing benchmark fails the run; - core forwards the child exit code (128 + N for signals) and the execArgv. Tests that drive a plugin in-process against a mocked core now set `CODSPEED_PROCESS_ISOLATION=false`, because isolation would start the test runner again for every benchmark. --- .../tests/fixtures/isolation.cjs | 37 ++++ .../tests/index.integ.test.ts | 4 + .../tests/isolation.integ.test.ts | 107 ++++++++++++ packages/core/tests/fixtures/isolation.cjs | 27 +++ packages/core/tests/isolation.integ.test.ts | 162 ++++++++++++++++++ .../tests/fixtures/isolation.mjs | 46 +++++ .../tests/index.integ.test.ts | 3 + .../tests/isolation.integ.test.ts | 122 +++++++++++++ .../src/__tests__/instrumented.test.ts | 6 + .../src/__tests__/isolation.test.ts | 113 ++++++++++++ .../tests/fixtures/isolation/pids.bench.ts | 19 ++ .../tests/fixtures/isolation/stub.mjs | 16 ++ .../tests/fixtures/isolation/vitest.config.ts | 12 ++ 13 files changed, 674 insertions(+) create mode 100644 packages/benchmark.js-plugin/tests/fixtures/isolation.cjs create mode 100644 packages/benchmark.js-plugin/tests/isolation.integ.test.ts create mode 100644 packages/core/tests/fixtures/isolation.cjs create mode 100644 packages/core/tests/isolation.integ.test.ts create mode 100644 packages/tinybench-plugin/tests/fixtures/isolation.mjs create mode 100644 packages/tinybench-plugin/tests/isolation.integ.test.ts create mode 100644 packages/vitest-plugin/src/__tests__/isolation.test.ts create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/stub.mjs create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts diff --git a/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs b/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs new file mode 100644 index 00000000..1ab23728 --- /dev/null +++ b/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs @@ -0,0 +1,37 @@ +// Run by `isolation.integ.test.ts`. Prints the PID that measures each benchmark. +const { InstrumentHooks } = require("@codspeed/core"); +const Benchmark = require("benchmark"); +const { withCodSpeed } = require("../../dist/index.cjs.js"); + +// There is no instrument hook runtime outside the CodSpeed runner, so pretend +// the hooks are active. The child processes run this file again, so they get +// the same override. +InstrumentHooks.isInstrumented = () => true; + +// The plugin reports every measured benchmark here, with the PID of the +// process that measured it. +const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark; +InstrumentHooks.setExecutedBenchmark = (pid, uri) => { + setExecutedBenchmark(pid, uri); + console.log(`executed ${pid} ${uri}`); +}; + +const suite = withCodSpeed(new Benchmark.Suite("isolation")); + +suite + .add("first", () => {}) + .add("second", () => {}) + .add("third", () => {}) + .add("async", { + defer: true, + fn: (deferred) => deferred.resolve(), + }); + +if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + suite.add("failing", () => { + throw new Error("benchmark failed"); + }); +} + +// An unhandled rejection ends the process with exit code 1. +suite.run(); diff --git a/packages/benchmark.js-plugin/tests/index.integ.test.ts b/packages/benchmark.js-plugin/tests/index.integ.test.ts index f3e21034..3d2256f5 100644 --- a/packages/benchmark.js-plugin/tests/index.integ.test.ts +++ b/packages/benchmark.js-plugin/tests/index.integ.test.ts @@ -11,6 +11,7 @@ jest.mock("@codspeed/core", () => { const actual = jest.requireActual("@codspeed/core"); mockCore.getGitDir = actual.getGitDir; mockCore.getCallingFile = actual.getCallingFile; + mockCore.Isolation = actual.Isolation; mockCore.wrapWithRootFrame = actual.wrapWithRootFrame; mockCore.wrapWithRootFrameSync = actual.wrapWithRootFrameSync; return mockCore; @@ -19,6 +20,9 @@ jest.mock("@codspeed/core", () => { beforeEach(() => { mockReset(mockCore); jest.clearAllMocks(); + // These tests drive the plugin in this process against a mocked core, so + // there is no benchmark process to spawn. + process.env.CODSPEED_PROCESS_ISOLATION = "false"; }); const benchOptions: Benchmark.Options = { diff --git a/packages/benchmark.js-plugin/tests/isolation.integ.test.ts b/packages/benchmark.js-plugin/tests/isolation.integ.test.ts new file mode 100644 index 00000000..0c59e91d --- /dev/null +++ b/packages/benchmark.js-plugin/tests/isolation.integ.test.ts @@ -0,0 +1,107 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawnSync } from "child_process"; +import { join } from "path"; + +const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +/** `first`, `second`, `third` and the deferred `async` benchmark. */ +const BENCHMARK_COUNT = 4; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const EXECUTED_LINE = /^executed (\d+) (.+)$/; + +interface FixtureRun { + status: number | null; + stderr: string; + /** PID that measured each benchmark. */ + pidsByUri: Map; + /** PID of the process the test started. */ + parentPid: number; +} + +interface FixtureOptions { + /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ + isolation?: string; + failing?: boolean; +} + +function runFixture({ isolation, failing }: FixtureOptions = {}): FixtureRun { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + }; + delete env[PROCESS_ISOLATION_ENV]; + if (isolation !== undefined) { + env[PROCESS_ISOLATION_ENV] = isolation; + } + if (failing) { + env.ISOLATION_FIXTURE_FAIL = "1"; + } + + // The runner starts Node with these flags, and children must keep them. + const result = spawnSync(process.execPath, [...getV8Flags(), FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + + const pidsByUri = new Map(); + for (const line of result.stdout.split("\n")) { + const match = EXECUTED_LINE.exec(line); + if (match) { + pidsByUri.set(match[2], Number(match[1])); + } + } + return { + status: result.status, + stderr: result.stderr, + pidsByUri, + parentPid: result.pid, + }; +} + +describe("process isolation", () => { + it.each([undefined, "1", "true"])( + "measures every benchmark in its own process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + // The parent only spawns children, it never measures. + expect(pids).not.toContain(run.parentPid); + }, + ); + + it.each(["0", "false"])( + "measures all benchmarks in one process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid])); + }, + ); + + it.each(["true", "false"])( + "fails the run when a benchmark throws (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, failing: true }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain("benchmark failed"); + }, + ); + + it("rejects values other than true and false", () => { + const run = runFixture({ isolation: "maybe" }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain(`${PROCESS_ISOLATION_ENV} must be`); + expect(run.pidsByUri.size).toBe(0); + }); +}); diff --git a/packages/core/tests/fixtures/isolation.cjs b/packages/core/tests/fixtures/isolation.cjs new file mode 100644 index 00000000..46205d3b --- /dev/null +++ b/packages/core/tests/fixtures/isolation.cjs @@ -0,0 +1,27 @@ +// Run by `isolation.integ.test.ts`. Runs two benchmarks through `Isolation`. +const { Isolation } = require("../../dist/index.cjs.js"); + +const isolation = Isolation.current(); + +function bench(uri) { + console.log(`ran ${process.pid} ${uri} ${JSON.stringify(process.execArgv)}`); + switch (process.env.ISOLATION_FIXTURE_ACTION) { + case "exit": + process.exit(3); + case "kill": + process.kill(process.pid, "SIGKILL"); + } +} + +async function main() { + for (const uri of ["a", "b"]) { + if (!isolation.shouldRun(uri)) continue; + if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { + isolation.runSync(uri, () => bench(uri)); + } else { + await isolation.run(uri, async () => bench(uri)); + } + } +} + +main(); diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts new file mode 100644 index 00000000..8649d8c5 --- /dev/null +++ b/packages/core/tests/isolation.integ.test.ts @@ -0,0 +1,162 @@ +import { spawnSync } from "child_process"; +import { constants } from "os"; +import { join } from "path"; +import { + ISOLATED_BENCHMARK_ENV, + Isolation, + PROCESS_ISOLATION_ENV, +} from "../src/isolation"; + +const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const SIGNAL_EXIT_CODE_OFFSET = 128; +const CHILD_EXIT_CODE = 3; +const RAN_LINE = /^ran (\d+) (\S+) (.+)$/; +/** Passed to Node to check that children keep the flags of their parent. */ +const NODE_FLAG = "--no-warnings"; + +/** `Isolation.current()` reads the environment once, so load a fresh copy. */ +function currentIsolation(env: Record): Isolation { + const saved = Object.fromEntries( + Object.keys(env).map((key) => [key, process.env[key]]), + ); + setEnv(env); + try { + let isolation: Isolation | undefined; + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + isolation = require("../src/isolation").Isolation.current(); + }); + return isolation as unknown as Isolation; + } finally { + setEnv(saved); + } +} + +function setEnv(env: Record): void { + for (const [key, value] of Object.entries(env)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } +} + +describe("Isolation.current", () => { + it.each([undefined, "1", "true"])( + "runs one process per benchmark (isolation=%s)", + (value) => { + const isolation = currentIsolation({ + [PROCESS_ISOLATION_ENV]: value, + [ISOLATED_BENCHMARK_ENV]: undefined, + }); + + expect(isolation.mode).toEqual({ kind: "ProcessPerBenchmark" }); + expect(isolation.onlyBenchmark()).toBeUndefined(); + expect(isolation.isChild()).toBe(false); + expect(isolation.shouldRun("any")).toBe(true); + }, + ); + + it.each(["0", "false"])("shares one process (isolation=%s)", (value) => { + const isolation = currentIsolation({ + [PROCESS_ISOLATION_ENV]: value, + [ISOLATED_BENCHMARK_ENV]: undefined, + }); + + expect(isolation.mode).toEqual({ kind: "SharedProcess" }); + expect(isolation.isChild()).toBe(false); + expect(isolation.shouldRun("any")).toBe(true); + }); + + it("restricts a child to its benchmark, whatever the isolation setting", () => { + const isolation = currentIsolation({ + [PROCESS_ISOLATION_ENV]: "false", + [ISOLATED_BENCHMARK_ENV]: "file::bench", + }); + + expect(isolation.mode).toEqual({ + kind: "IsolatedBenchmark", + uri: "file::bench", + }); + expect(isolation.onlyBenchmark()).toBe("file::bench"); + expect(isolation.isChild()).toBe(true); + expect(isolation.shouldRun("file::bench")).toBe(true); + // Exact match only: a prefix is another benchmark. + expect(isolation.shouldRun("file::bench 2")).toBe(false); + expect(isolation.shouldRun("file")).toBe(false); + }); + + it("rejects any other value", () => { + expect(() => + currentIsolation({ + [PROCESS_ISOLATION_ENV]: "maybe", + [ISOLATED_BENCHMARK_ENV]: undefined, + }), + ).toThrow(`${PROCESS_ISOLATION_ENV} must be \`true\` or \`false\``); + }); + + it("never spawns when no instrument measures", () => { + expect(Isolation.shared().mode).toEqual({ kind: "SharedProcess" }); + }); +}); + +interface FixtureOptions { + action?: "exit" | "kill"; + sync?: boolean; +} + +function runFixture({ action, sync }: FixtureOptions = {}) { + const env: NodeJS.ProcessEnv = { + ...process.env, + ISOLATION_FIXTURE_ACTION: action, + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[ISOLATED_BENCHMARK_ENV]; + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + const result = spawnSync(process.execPath, [NODE_FLAG, FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + const ran = result.stdout + .split("\n") + .map((line) => RAN_LINE.exec(line)) + .filter((match): match is RegExpExecArray => match !== null); + return { status: result.status, ran }; +} + +describe.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, +])("Isolation child process ($name run)", ({ sync }) => { + it("runs each benchmark in a fresh process with the flags of its parent", () => { + const { status, ran } = runFixture({ sync }); + + expect(status).toBe(0); + expect(ran.map((match) => match[2])).toEqual(["a", "b"]); + expect(new Set(ran.map((match) => match[1])).size).toBe(2); + for (const match of ran) { + expect(JSON.parse(match[3])).toEqual([NODE_FLAG]); + } + }); + + it("exits with the exit code of a failing child", () => { + const { status, ran } = runFixture({ sync, action: "exit" }); + + expect(status).toBe(CHILD_EXIT_CODE); + // The parent stops at the first failure. + expect(ran.map((match) => match[2])).toEqual(["a"]); + }); + + it("exits with 128 + N when a signal N killed the child", () => { + const { status, ran } = runFixture({ sync, action: "kill" }); + + expect(status).toBe(SIGNAL_EXIT_CODE_OFFSET + constants.signals.SIGKILL); + expect(ran.map((match) => match[2])).toEqual(["a"]); + }); +}); diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs new file mode 100644 index 00000000..ec2424da --- /dev/null +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -0,0 +1,46 @@ +// Run by `isolation.integ.test.ts`. Prints the PID that measures each benchmark. +import { InstrumentHooks } from "@codspeed/core"; +import { Bench } from "tinybench"; +import { withCodSpeed } from "../../dist/index.es5.js"; + +// There is no instrument hook runtime outside the CodSpeed runner, so pretend +// the hooks are active. The child processes run this file again, so they get +// the same override. +InstrumentHooks.isInstrumented = () => true; + +// The plugin reports every measured benchmark here, with the PID of the +// process that measured it. +const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark; +InstrumentHooks.setExecutedBenchmark = (pid, uri) => { + setExecutedBenchmark(pid, uri); + console.log(`executed ${pid} ${uri}`); +}; + +// tinybench calls each function once when it is added, so a failing benchmark +// may only throw once the measurement started. +let measuring = false; +const startBenchmark = InstrumentHooks.startBenchmark; +InstrumentHooks.startBenchmark = () => { + measuring = true; + startBenchmark(); +}; + +const bench = withCodSpeed(new Bench({ time: 1, warmupTime: 0 })); + +for (const name of ["first", "second", "third"]) { + bench.add(name, () => {}); +} + +if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + bench.add("failing", () => { + if (measuring) { + throw new Error("benchmark failed"); + } + }); +} + +if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { + bench.runSync(); +} else { + await bench.run(); +} diff --git a/packages/tinybench-plugin/tests/index.integ.test.ts b/packages/tinybench-plugin/tests/index.integ.test.ts index 5e71eeb2..6de91c6e 100644 --- a/packages/tinybench-plugin/tests/index.integ.test.ts +++ b/packages/tinybench-plugin/tests/index.integ.test.ts @@ -41,6 +41,9 @@ vi.mock("@codspeed/core", async (importOriginal) => { beforeEach(() => { process.env.CODSPEED_ENV = "true"; process.env.CODSPEED_RUNNER_MODE = "instrumentation"; + // These tests drive the plugin in this process against a mocked core, so + // there is no benchmark process to spawn. + process.env.CODSPEED_PROCESS_ISOLATION = "false"; vi.clearAllMocks(); }); diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts new file mode 100644 index 00000000..27970175 --- /dev/null +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -0,0 +1,122 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawnSync } from "child_process"; +import { fileURLToPath } from "url"; +import { describe, expect, it } from "vitest"; + +const FIXTURE = fileURLToPath( + new URL("./fixtures/isolation.mjs", import.meta.url), +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const BENCHMARK_COUNT = 3; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const EXECUTED_LINE = /^executed (\d+) (.+)$/; + +interface FixtureRun { + status: number | null; + stderr: string; + /** PID that measured each benchmark, in the order they were measured. */ + pidsByUri: Map; + /** PID of the process the test started. */ + parentPid: number; +} + +interface FixtureOptions { + /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ + isolation?: string; + failing?: boolean; + sync?: boolean; +} + +function runFixture({ + isolation, + failing, + sync, +}: FixtureOptions = {}): FixtureRun { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + }; + delete env[PROCESS_ISOLATION_ENV]; + if (isolation !== undefined) { + env[PROCESS_ISOLATION_ENV] = isolation; + } + if (failing) { + env.ISOLATION_FIXTURE_FAIL = "1"; + } + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + + // The runner starts Node with these flags, and children must keep them. + const result = spawnSync(process.execPath, [...getV8Flags(), FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + + const pidsByUri = new Map(); + for (const line of result.stdout.split("\n")) { + const match = EXECUTED_LINE.exec(line); + if (match) { + pidsByUri.set(match[2], Number(match[1])); + } + } + return { + status: result.status, + stderr: result.stderr, + pidsByUri, + parentPid: result.pid, + }; +} + +describe.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, +])("process isolation ($name run)", ({ sync }) => { + it.each([undefined, "1", "true"])( + "measures every benchmark in its own process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, sync }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + // The parent only spawns children, it never measures. + expect(pids).not.toContain(run.parentPid); + }, + ); + + it.each(["0", "false"])( + "measures all benchmarks in one process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, sync }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid])); + }, + ); + + it.each(["true", "false"])( + "fails the run when a benchmark throws (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, failing: true, sync }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain("benchmark failed"); + }, + ); +}); + +describe("process isolation setting", () => { + it("rejects values other than true and false", () => { + const run = runFixture({ isolation: "maybe" }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain(`${PROCESS_ISOLATION_ENV} must be`); + expect(run.pidsByUri.size).toBe(0); + }); +}); diff --git a/packages/vitest-plugin/src/__tests__/instrumented.test.ts b/packages/vitest-plugin/src/__tests__/instrumented.test.ts index ec1a1fd9..2e7f54c6 100644 --- a/packages/vitest-plugin/src/__tests__/instrumented.test.ts +++ b/packages/vitest-plugin/src/__tests__/instrumented.test.ts @@ -6,6 +6,8 @@ import { getBenchFn, getBenchOptions } from "../legacy/compat"; const coreMocks = vi.hoisted(() => { return { InstrumentHooks: { + // A plain function: the mocks above are reset between tests. + isInstrumented: () => true, startBenchmark: vi.fn(), stopBenchmark: vi.fn(), setExecutedBenchmark: vi.fn(), @@ -21,6 +23,10 @@ const coreMocks = vi.hoisted(() => { global.eval = vi.fn(); +// The runner is driven in this process against a mocked core, so there is no +// benchmark process to spawn. +process.env.CODSPEED_PROCESS_ISOLATION = "false"; + vi.mock("@codspeed/core", async (importOriginal) => { const mod = await importOriginal(); return { ...mod, ...coreMocks }; diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts new file mode 100644 index 00000000..6683ccf1 --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -0,0 +1,113 @@ +import { spawnSync } from "child_process"; +import { mkdtempSync, readFileSync, rmSync } from "fs"; +import { createRequire } from "module"; +import { tmpdir } from "os"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const FIXTURE_DIR = fileURLToPath( + new URL("../../tests/fixtures/isolation", import.meta.url), +); +const PRELOAD = join(FIXTURE_DIR, "stub.mjs"); +const VITEST_BIN = join( + dirname(createRequire(import.meta.url).resolve("vitest/package.json")), + "vitest.mjs", +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const BENCHMARK_COUNT = 3; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 120_000; +const EXECUTED_LINE = /^executed (\d+) (.+)$/; + +let logDir: string; +let logFile: string; + +beforeEach(() => { + logDir = mkdtempSync(join(tmpdir(), "codspeed-isolation-")); + logFile = join(logDir, "executed.log"); +}); + +afterEach(() => { + rmSync(logDir, { recursive: true, force: true }); +}); + +interface FixtureOptions { + /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ + isolation?: string; + failing?: boolean; +} + +/** Runs `vitest bench` on the fixture project and returns what it measured. */ +function runFixture({ isolation, failing }: FixtureOptions = {}) { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + ISOLATION_LOG_FILE: logFile, + NODE_OPTIONS: [process.env.NODE_OPTIONS, `--import ${PRELOAD}`] + .filter(Boolean) + .join(" "), + }; + delete env[PROCESS_ISOLATION_ENV]; + if (isolation !== undefined) { + env[PROCESS_ISOLATION_ENV] = isolation; + } + if (failing) { + env.ISOLATION_FIXTURE_FAIL = "1"; + } + + const result = spawnSync(process.execPath, [VITEST_BIN, "bench", "--run"], { + cwd: FIXTURE_DIR, + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + + const pidsByUri = new Map(); + for (const line of readFileSync(logFile, "utf8").split("\n")) { + const match = EXECUTED_LINE.exec(line); + if (match) { + pidsByUri.set(match[2], Number(match[1])); + } + } + return { status: result.status, pidsByUri, parentPid: result.pid }; +} + +describe("process isolation", () => { + it.each([undefined, "1", "true"])( + "measures every benchmark in its own process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + expect(pids).not.toContain(run.parentPid); + }, + RUN_TIMEOUT_MS, + ); + + it.each(["0", "false"])( + "measures all benchmarks in one process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values()).size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it.each(["true", "false"])( + "fails the run when a benchmark throws (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, failing: true }); + + expect(run.status).not.toBe(0); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts new file mode 100644 index 00000000..c803dcbc --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts @@ -0,0 +1,19 @@ +import { describe, test } from "vitest"; + +describe("isolation", () => { + test("benchmarks", async ({ bench }) => { + const benchmarks = [ + bench("first", () => {}), + bench("second", () => {}), + bench("third", () => {}), + ]; + if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + benchmarks.push( + bench("failing", () => { + throw new Error("benchmark failed"); + }), + ); + } + await bench.compare(...benchmarks); + }); +}); diff --git a/packages/vitest-plugin/tests/fixtures/isolation/stub.mjs b/packages/vitest-plugin/tests/fixtures/isolation/stub.mjs new file mode 100644 index 00000000..bdfd51b6 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/stub.mjs @@ -0,0 +1,16 @@ +// Preloaded with `node --import` into the Vitest process, its workers, and the +// Vitest runs that process isolation starts. +import { InstrumentHooks } from "@codspeed/core"; +import { appendFileSync } from "fs"; + +// There is no instrument hook runtime outside the CodSpeed runner, so pretend +// the hooks are active. +InstrumentHooks.isInstrumented = () => true; + +// Vitest swallows what workers print, so the PID that measures each benchmark +// goes to a file. +const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark; +InstrumentHooks.setExecutedBenchmark = (pid, uri) => { + setExecutedBenchmark(pid, uri); + appendFileSync(process.env.ISOLATION_LOG_FILE, `executed ${pid} ${uri}\n`); +}; diff --git a/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts new file mode 100644 index 00000000..df313ac4 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; +import codspeedPlugin from "../../../dist/index.mjs"; + +export default defineConfig({ + plugins: [codspeedPlugin()], + define: { + __VERSION__: JSON.stringify("1.0.0"), + }, + test: { + include: ["**/*.bench.ts"], + }, +}); From 4bd0611e6514c9040f81fd455673e77c51969218 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:45 +0200 Subject: [PATCH 6/6] docs: describe process isolation Document the environment contract, the parent and child roles, and the Vitest and test caveats. --- CLAUDE.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 00a76a53..ab1d6183 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -115,4 +115,26 @@ so exercise all three of these when developing or reviewing a plugin change Running these locally outside the CodSpeed runner is expected to log `instrument-hooks: failed to write environment.json` and skip actual measurement writes — the point is to verify the plugin's control flow and output per mode, -not to produce real measurements. \ No newline at end of file +not to produce real measurements. + +## Process isolation + +In instrumentation/simulation/memory runs with active instrument hooks, every +benchmark runs in its own process (`packages/core/src/isolation.ts`), so the +allocator and JIT state of one benchmark can't change the measurement of the +next. Walltime runs and runs outside CodSpeed are not affected. + +- `CODSPEED_PROCESS_ISOLATION`: unset, `1` or `true` turns it on (default), + `0` or `false` runs all benchmarks in one process, any other value is an error. +- The parent process re-runs itself (`process.execPath`, `process.execArgv` and + `process.argv.slice(1)`, so the V8 flags survive) once per benchmark with + `CODSPEED_ISOLATED_BENCHMARK=`. The child runs only that benchmark, whatever + the user filters are, and the parent exits with the child's exit code + (`128 + N` for signal `N`). +- The child prints no banner or summary lines. They come from the parent. +- Vitest measures in forked workers, so the plugin records the command of the + main Vitest process (`CODSPEED_ISOLATION_COMMAND`) and each worker runs that + command, i.e. one Vitest run per benchmark. +- Tests that drive a plugin in-process against a mocked core must set + `CODSPEED_PROCESS_ISOLATION=false`, or the plugin would start the test runner + again for every benchmark. \ No newline at end of file