feat: run each benchmark in its own process

CLAUDE.md

@@ -115,4 +115,26 @@ so exercise all three of these when developing or reviewing a plugin change
 Running these locally outside the CodSpeed runner is expected to log
 `instrument-hooks: failed to write environment.json` and skip actual measurement
 writes — the point is to verify the plugin's control flow and output per mode,
-not to produce real measurements.
+not to produce real measurements.
+
+## Process isolation
+
+In instrumentation/simulation/memory runs with active instrument hooks, every
+benchmark runs in its own process (`packages/core/src/isolation.ts`), so the
+allocator and JIT state of one benchmark can't change the measurement of the
+next. Walltime runs and runs outside CodSpeed are not affected.
+
+- `CODSPEED_PROCESS_ISOLATION`: unset, `1` or `true` turns it on (default),
+  `0` or `false` runs all benchmarks in one process, any other value is an error.
+- The parent process re-runs itself (`process.execPath`, `process.execArgv` and
+  `process.argv.slice(1)`, so the V8 flags survive) once per benchmark with
+  `CODSPEED_ISOLATED_BENCHMARK=<uri>`. The child runs only that benchmark, whatever
+  the user filters are, and the parent exits with the child's exit code
+  (`128 + N` for signal `N`).
+- The child prints no banner or summary lines. They come from the parent.
+- Vitest measures in forked workers, so the plugin records the command of the
+  main Vitest process (`CODSPEED_ISOLATION_COMMAND`) and each worker runs that
+  command, i.e. one Vitest run per benchmark.
+- Tests that drive a plugin in-process against a mocked core must set
+  `CODSPEED_PROCESS_ISOLATION=false`, or the plugin would start the test runner
+  again for every benchmark.

packages/benchmark.js-plugin/src/index.ts

@@ -1,6 +1,7 @@
 import {
   getCallingFile,
   InstrumentHooks,
+  Isolation,
   mongoMeasurement,
   optimizeFunction,
   optimizeFunctionSync,
@@ -168,63 +169,92 @@ async function runBenchmarks({
   baseUri,
   benchmarkCompletedListeners,
 }: RunBenchmarksOptions): Promise<void> {
-  console.log(`[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`);
+  const isolation = Isolation.current();
+  const isChild = isolation.isChild();
+  const selected = benches
+    .map((bench, i) => ({
+      bench,
+      uri: bench.uri ?? `${baseUri}::unknown_${i}`,
+    }))
+    .filter(({ uri }) => isolation.shouldRun(uri));
+  // A child only measures one benchmark. A run call that owns none of them
+  // must not touch the instruments, the call that owns it does.
+  if (isChild && selected.length === 0) {
+    return;
+  }
+
+  // The parent already printed the banner and the summary, children stay quiet.
+  if (!isChild) {
+    console.log(
+      `[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`,
+    );
+  }
   setupCore();
-  for (let i = 0; i < benches.length; i++) {
-    const bench = benches[i];
-    const uri = bench.uri ?? `${baseUri}::unknown_${i}`;
-    const isAsync = bench.options.async || bench.options.defer;
-    let benchPayload;
-    if (bench.options.defer) {
-      benchPayload = () => {
-        return new Promise((resolve, reject) => {
-          (bench.fn as CallableFunction)({ resolve, reject });
-        });
-      };
-    } else if (bench.options.async) {
-      benchPayload = async () => {
-        await (bench.fn as CallableFunction)();
-      };
-    } else {
-      benchPayload = bench.fn as CallableFunction;
+  for (const { bench, uri } of selected) {
+    await isolation.run(uri, () => measureBenchmark(bench, uri));
+    // The parent reports for every benchmark, the child ran only one of them.
+    if (!isChild) {
+      benchmarkCompletedListeners.forEach((listener) => listener());
     }
+  }
+  teardownCore();
+  if (!isChild) {
+    console.log(`[CodSpeed] Done running ${benches.length} benches.`);
+  }
+}
 
-    if (typeof bench.options.setup === "function") {
-      await bench.options.setup();
-    }
+async function measureBenchmark(
+  bench: BenchmarkWithOptions,
+  uri: string,
+): Promise<void> {
+  const isAsync = bench.options.async || bench.options.defer;
+  let benchPayload;
+  if (bench.options.defer) {
+    benchPayload = () => {
+      const { promise, resolve, reject } = Promise.withResolvers<unknown>();
+      (bench.fn as CallableFunction)({ resolve, reject });
+      return promise;
+    };
+  } else if (bench.options.async) {
+    benchPayload = async () => {
+      await (bench.fn as CallableFunction)();
+    };
+  } else {
+    benchPayload = bench.fn as CallableFunction;
+  }
 
-    if (isAsync) {
-      await optimizeFunction(benchPayload);
-      await mongoMeasurement.start(uri);
-      global.gc?.();
-      await wrapWithRootFrame(async () => {
-        InstrumentHooks.startBenchmark();
-        await benchPayload();
-        InstrumentHooks.stopBenchmark();
-        InstrumentHooks.setExecutedBenchmark(process.pid, uri);
-      })();
-      await mongoMeasurement.stop(uri);
-    } else {
-      optimizeFunctionSync(benchPayload);
-      await mongoMeasurement.start(uri);
-      wrapWithRootFrameSync(() => {
-        InstrumentHooks.startBenchmark();
-        benchPayload();
-        InstrumentHooks.stopBenchmark();
-        InstrumentHooks.setExecutedBenchmark(process.pid, uri);
-      })();
-      await mongoMeasurement.stop(uri);
-    }
+  if (typeof bench.options.setup === "function") {
+    await bench.options.setup();
+  }
 
-    if (typeof bench.options.teardown === "function") {
-      await bench.options.teardown();
-    }
+  if (isAsync) {
+    await optimizeFunction(benchPayload);
+    await mongoMeasurement.start(uri);
+    global.gc?.();
+    await wrapWithRootFrame(async () => {
+      InstrumentHooks.startBenchmark();
+      await benchPayload();
+      InstrumentHooks.stopBenchmark();
+      InstrumentHooks.setExecutedBenchmark(process.pid, uri);
+    })();
+    await mongoMeasurement.stop(uri);
+  } else {
+    optimizeFunctionSync(benchPayload);
+    await mongoMeasurement.start(uri);
+    wrapWithRootFrameSync(() => {
+      InstrumentHooks.startBenchmark();
+      benchPayload();
+      InstrumentHooks.stopBenchmark();
+      InstrumentHooks.setExecutedBenchmark(process.pid, uri);
+    })();
+    await mongoMeasurement.stop(uri);
+  }
 
-    console.log(`    ✔ Measured ${uri}`);
-    benchmarkCompletedListeners.forEach((listener) => listener());
+  if (typeof bench.options.teardown === "function") {
+    await bench.options.teardown();
   }
-  teardownCore();
-  console.log(`[CodSpeed] Done running ${benches.length} benches.`);
+
+  console.log(`    ✔ Measured ${uri}`);
 }
 
 /**

packages/benchmark.js-plugin/tests/fixtures/isolation.cjs

@@ -0,0 +1,37 @@
+// Run by `isolation.integ.test.ts`. Prints the PID that measures each benchmark.
+const { InstrumentHooks } = require("@codspeed/core");
+const Benchmark = require("benchmark");
+const { withCodSpeed } = require("../../dist/index.cjs.js");
+
+// There is no instrument hook runtime outside the CodSpeed runner, so pretend
+// the hooks are active. The child processes run this file again, so they get
+// the same override.
+InstrumentHooks.isInstrumented = () => true;
+
+// The plugin reports every measured benchmark here, with the PID of the
+// process that measured it.
+const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark;
+InstrumentHooks.setExecutedBenchmark = (pid, uri) => {
+  setExecutedBenchmark(pid, uri);
+  console.log(`executed ${pid} ${uri}`);
+};
+
+const suite = withCodSpeed(new Benchmark.Suite("isolation"));
+
+suite
+  .add("first", () => {})
+  .add("second", () => {})
+  .add("third", () => {})
+  .add("async", {
+    defer: true,
+    fn: (deferred) => deferred.resolve(),
+  });
+
+if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) {
+  suite.add("failing", () => {
+    throw new Error("benchmark failed");
+  });
+}
+
+// An unhandled rejection ends the process with exit code 1.
+suite.run();

packages/benchmark.js-plugin/tests/index.integ.test.ts

@@ -11,6 +11,7 @@ jest.mock("@codspeed/core", () => {
   const actual = jest.requireActual("@codspeed/core");
   mockCore.getGitDir = actual.getGitDir;
   mockCore.getCallingFile = actual.getCallingFile;
+  mockCore.Isolation = actual.Isolation;
   mockCore.wrapWithRootFrame = actual.wrapWithRootFrame;
   mockCore.wrapWithRootFrameSync = actual.wrapWithRootFrameSync;
   return mockCore;
@@ -19,6 +20,9 @@ jest.mock("@codspeed/core", () => {
 beforeEach(() => {
   mockReset(mockCore);
   jest.clearAllMocks();
+  // These tests drive the plugin in this process against a mocked core, so
+  // there is no benchmark process to spawn.
+  process.env.CODSPEED_PROCESS_ISOLATION = "false";
 });
 
 const benchOptions: Benchmark.Options = {

packages/benchmark.js-plugin/tests/isolation.integ.test.ts

@@ -0,0 +1,107 @@
+import { getV8Flags } from "@codspeed/core";
+import { spawnSync } from "child_process";
+import { join } from "path";
+
+const FIXTURE = join(__dirname, "fixtures", "isolation.cjs");
+const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION";
+/** `first`, `second`, `third` and the deferred `async` benchmark. */
+const BENCHMARK_COUNT = 4;
+/** Fails a run whose benchmark process got stuck instead of blocking the suite. */
+const RUN_TIMEOUT_MS = 60_000;
+const EXECUTED_LINE = /^executed (\d+) (.+)$/;
+
+interface FixtureRun {
+  status: number | null;
+  stderr: string;
+  /** PID that measured each benchmark. */
+  pidsByUri: Map<string, number>;
+  /** PID of the process the test started. */
+  parentPid: number;
+}
+
+interface FixtureOptions {
+  /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */
+  isolation?: string;
+  failing?: boolean;
+}
+
+function runFixture({ isolation, failing }: FixtureOptions = {}): FixtureRun {
+  const env: NodeJS.ProcessEnv = {
+    ...process.env,
+    CODSPEED_ENV: "true",
+    CODSPEED_RUNNER_MODE: "simulation",
+  };
+  delete env[PROCESS_ISOLATION_ENV];
+  if (isolation !== undefined) {
+    env[PROCESS_ISOLATION_ENV] = isolation;
+  }
+  if (failing) {
+    env.ISOLATION_FIXTURE_FAIL = "1";
+  }
+
+  // The runner starts Node with these flags, and children must keep them.
+  const result = spawnSync(process.execPath, [...getV8Flags(), FIXTURE], {
+    env,
+    encoding: "utf8",
+    timeout: RUN_TIMEOUT_MS,
+  });
+
+  const pidsByUri = new Map<string, number>();
+  for (const line of result.stdout.split("\n")) {
+    const match = EXECUTED_LINE.exec(line);
+    if (match) {
+      pidsByUri.set(match[2], Number(match[1]));
+    }
+  }
+  return {
+    status: result.status,
+    stderr: result.stderr,
+    pidsByUri,
+    parentPid: result.pid,
+  };
+}
+
+describe("process isolation", () => {
+  it.each([undefined, "1", "true"])(
+    "measures every benchmark in its own process (isolation=%s)",
+    (isolation) => {
+      const run = runFixture({ isolation });
+
+      expect(run.status).toBe(0);
+      expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT);
+      const pids = new Set(run.pidsByUri.values());
+      expect(pids.size).toBe(BENCHMARK_COUNT);
+      // The parent only spawns children, it never measures.
+      expect(pids).not.toContain(run.parentPid);
+    },
+  );
+
+  it.each(["0", "false"])(
+    "measures all benchmarks in one process (isolation=%s)",
+    (isolation) => {
+      const run = runFixture({ isolation });
+
+      expect(run.status).toBe(0);
+      expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT);
+      expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid]));
+    },
+  );
+
+  it.each(["true", "false"])(
+    "fails the run when a benchmark throws (isolation=%s)",
+    (isolation) => {
+      const run = runFixture({ isolation, failing: true });
+
+      expect(run.status).not.toBe(0);
+      expect(run.stderr).toContain("benchmark failed");
+    },
+  );
+
+  it("rejects values other than true and false", () => {
+    const run = runFixture({ isolation: "maybe" });
+
+    expect(run.status).not.toBe(0);
+    expect(run.stderr).toContain(`${PROCESS_ISOLATION_ENV} must be`);
+    expect(run.pidsByUri.size).toBe(0);
+  });
+});

packages/core/src/index.ts

@@ -51,6 +51,7 @@ export type {
   SetupInstrumentsResponse,
 } from "./generated/openapi";
 export { getV8Flags, tryIntrospect } from "./introspection";
+export * from "./isolation";
 export {
   getUnsupportedNodeVersionWarning,
   SUPPORTED_NODE_MAJORS,