diff --git a/CLAUDE.md b/CLAUDE.md index 00a76a53..ab1d6183 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -115,4 +115,26 @@ so exercise all three of these when developing or reviewing a plugin change Running these locally outside the CodSpeed runner is expected to log `instrument-hooks: failed to write environment.json` and skip actual measurement writes — the point is to verify the plugin's control flow and output per mode, -not to produce real measurements. \ No newline at end of file +not to produce real measurements. + +## Process isolation + +In instrumentation/simulation/memory runs with active instrument hooks, every +benchmark runs in its own process (`packages/core/src/isolation.ts`), so the +allocator and JIT state of one benchmark can't change the measurement of the +next. Walltime runs and runs outside CodSpeed are not affected. + +- `CODSPEED_PROCESS_ISOLATION`: unset, `1` or `true` turns it on (default), + `0` or `false` runs all benchmarks in one process, any other value is an error. +- The parent process re-runs itself (`process.execPath`, `process.execArgv` and + `process.argv.slice(1)`, so the V8 flags survive) once per benchmark with + `CODSPEED_ISOLATED_BENCHMARK=`. The child runs only that benchmark, whatever + the user filters are, and the parent exits with the child's exit code + (`128 + N` for signal `N`). +- The child prints no banner or summary lines. They come from the parent. +- Vitest measures in forked workers, so the plugin records the command of the + main Vitest process (`CODSPEED_ISOLATION_COMMAND`) and each worker runs that + command, i.e. one Vitest run per benchmark. +- Tests that drive a plugin in-process against a mocked core must set + `CODSPEED_PROCESS_ISOLATION=false`, or the plugin would start the test runner + again for every benchmark. \ No newline at end of file diff --git a/packages/benchmark.js-plugin/src/index.ts b/packages/benchmark.js-plugin/src/index.ts index fa69a35a..747b5fc2 100644 --- a/packages/benchmark.js-plugin/src/index.ts +++ b/packages/benchmark.js-plugin/src/index.ts @@ -1,6 +1,7 @@ import { getCallingFile, InstrumentHooks, + Isolation, mongoMeasurement, optimizeFunction, optimizeFunctionSync, @@ -168,63 +169,92 @@ async function runBenchmarks({ baseUri, benchmarkCompletedListeners, }: RunBenchmarksOptions): Promise { - console.log(`[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`); + const isolation = Isolation.current(); + const isChild = isolation.isChild(); + const selected = benches + .map((bench, i) => ({ + bench, + uri: bench.uri ?? `${baseUri}::unknown_${i}`, + })) + .filter(({ uri }) => isolation.shouldRun(uri)); + // A child only measures one benchmark. A run call that owns none of them + // must not touch the instruments, the call that owns it does. + if (isChild && selected.length === 0) { + return; + } + + // The parent already printed the banner and the summary, children stay quiet. + if (!isChild) { + console.log( + `[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`, + ); + } setupCore(); - for (let i = 0; i < benches.length; i++) { - const bench = benches[i]; - const uri = bench.uri ?? `${baseUri}::unknown_${i}`; - const isAsync = bench.options.async || bench.options.defer; - let benchPayload; - if (bench.options.defer) { - benchPayload = () => { - return new Promise((resolve, reject) => { - (bench.fn as CallableFunction)({ resolve, reject }); - }); - }; - } else if (bench.options.async) { - benchPayload = async () => { - await (bench.fn as CallableFunction)(); - }; - } else { - benchPayload = bench.fn as CallableFunction; + for (const { bench, uri } of selected) { + await isolation.run(uri, () => measureBenchmark(bench, uri)); + // The parent reports for every benchmark, the child ran only one of them. + if (!isChild) { + benchmarkCompletedListeners.forEach((listener) => listener()); } + } + teardownCore(); + if (!isChild) { + console.log(`[CodSpeed] Done running ${benches.length} benches.`); + } +} - if (typeof bench.options.setup === "function") { - await bench.options.setup(); - } +async function measureBenchmark( + bench: BenchmarkWithOptions, + uri: string, +): Promise { + const isAsync = bench.options.async || bench.options.defer; + let benchPayload; + if (bench.options.defer) { + benchPayload = () => { + const { promise, resolve, reject } = Promise.withResolvers(); + (bench.fn as CallableFunction)({ resolve, reject }); + return promise; + }; + } else if (bench.options.async) { + benchPayload = async () => { + await (bench.fn as CallableFunction)(); + }; + } else { + benchPayload = bench.fn as CallableFunction; + } - if (isAsync) { - await optimizeFunction(benchPayload); - await mongoMeasurement.start(uri); - global.gc?.(); - await wrapWithRootFrame(async () => { - InstrumentHooks.startBenchmark(); - await benchPayload(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - } else { - optimizeFunctionSync(benchPayload); - await mongoMeasurement.start(uri); - wrapWithRootFrameSync(() => { - InstrumentHooks.startBenchmark(); - benchPayload(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - } + if (typeof bench.options.setup === "function") { + await bench.options.setup(); + } - if (typeof bench.options.teardown === "function") { - await bench.options.teardown(); - } + if (isAsync) { + await optimizeFunction(benchPayload); + await mongoMeasurement.start(uri); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await benchPayload(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await mongoMeasurement.stop(uri); + } else { + optimizeFunctionSync(benchPayload); + await mongoMeasurement.start(uri); + wrapWithRootFrameSync(() => { + InstrumentHooks.startBenchmark(); + benchPayload(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await mongoMeasurement.stop(uri); + } - console.log(` ✔ Measured ${uri}`); - benchmarkCompletedListeners.forEach((listener) => listener()); + if (typeof bench.options.teardown === "function") { + await bench.options.teardown(); } - teardownCore(); - console.log(`[CodSpeed] Done running ${benches.length} benches.`); + + console.log(` ✔ Measured ${uri}`); } /** diff --git a/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs b/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs new file mode 100644 index 00000000..1ab23728 --- /dev/null +++ b/packages/benchmark.js-plugin/tests/fixtures/isolation.cjs @@ -0,0 +1,37 @@ +// Run by `isolation.integ.test.ts`. Prints the PID that measures each benchmark. +const { InstrumentHooks } = require("@codspeed/core"); +const Benchmark = require("benchmark"); +const { withCodSpeed } = require("../../dist/index.cjs.js"); + +// There is no instrument hook runtime outside the CodSpeed runner, so pretend +// the hooks are active. The child processes run this file again, so they get +// the same override. +InstrumentHooks.isInstrumented = () => true; + +// The plugin reports every measured benchmark here, with the PID of the +// process that measured it. +const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark; +InstrumentHooks.setExecutedBenchmark = (pid, uri) => { + setExecutedBenchmark(pid, uri); + console.log(`executed ${pid} ${uri}`); +}; + +const suite = withCodSpeed(new Benchmark.Suite("isolation")); + +suite + .add("first", () => {}) + .add("second", () => {}) + .add("third", () => {}) + .add("async", { + defer: true, + fn: (deferred) => deferred.resolve(), + }); + +if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + suite.add("failing", () => { + throw new Error("benchmark failed"); + }); +} + +// An unhandled rejection ends the process with exit code 1. +suite.run(); diff --git a/packages/benchmark.js-plugin/tests/index.integ.test.ts b/packages/benchmark.js-plugin/tests/index.integ.test.ts index f3e21034..3d2256f5 100644 --- a/packages/benchmark.js-plugin/tests/index.integ.test.ts +++ b/packages/benchmark.js-plugin/tests/index.integ.test.ts @@ -11,6 +11,7 @@ jest.mock("@codspeed/core", () => { const actual = jest.requireActual("@codspeed/core"); mockCore.getGitDir = actual.getGitDir; mockCore.getCallingFile = actual.getCallingFile; + mockCore.Isolation = actual.Isolation; mockCore.wrapWithRootFrame = actual.wrapWithRootFrame; mockCore.wrapWithRootFrameSync = actual.wrapWithRootFrameSync; return mockCore; @@ -19,6 +20,9 @@ jest.mock("@codspeed/core", () => { beforeEach(() => { mockReset(mockCore); jest.clearAllMocks(); + // These tests drive the plugin in this process against a mocked core, so + // there is no benchmark process to spawn. + process.env.CODSPEED_PROCESS_ISOLATION = "false"; }); const benchOptions: Benchmark.Options = { diff --git a/packages/benchmark.js-plugin/tests/isolation.integ.test.ts b/packages/benchmark.js-plugin/tests/isolation.integ.test.ts new file mode 100644 index 00000000..0c59e91d --- /dev/null +++ b/packages/benchmark.js-plugin/tests/isolation.integ.test.ts @@ -0,0 +1,107 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawnSync } from "child_process"; +import { join } from "path"; + +const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +/** `first`, `second`, `third` and the deferred `async` benchmark. */ +const BENCHMARK_COUNT = 4; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const EXECUTED_LINE = /^executed (\d+) (.+)$/; + +interface FixtureRun { + status: number | null; + stderr: string; + /** PID that measured each benchmark. */ + pidsByUri: Map; + /** PID of the process the test started. */ + parentPid: number; +} + +interface FixtureOptions { + /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ + isolation?: string; + failing?: boolean; +} + +function runFixture({ isolation, failing }: FixtureOptions = {}): FixtureRun { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + }; + delete env[PROCESS_ISOLATION_ENV]; + if (isolation !== undefined) { + env[PROCESS_ISOLATION_ENV] = isolation; + } + if (failing) { + env.ISOLATION_FIXTURE_FAIL = "1"; + } + + // The runner starts Node with these flags, and children must keep them. + const result = spawnSync(process.execPath, [...getV8Flags(), FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + + const pidsByUri = new Map(); + for (const line of result.stdout.split("\n")) { + const match = EXECUTED_LINE.exec(line); + if (match) { + pidsByUri.set(match[2], Number(match[1])); + } + } + return { + status: result.status, + stderr: result.stderr, + pidsByUri, + parentPid: result.pid, + }; +} + +describe("process isolation", () => { + it.each([undefined, "1", "true"])( + "measures every benchmark in its own process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + // The parent only spawns children, it never measures. + expect(pids).not.toContain(run.parentPid); + }, + ); + + it.each(["0", "false"])( + "measures all benchmarks in one process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid])); + }, + ); + + it.each(["true", "false"])( + "fails the run when a benchmark throws (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, failing: true }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain("benchmark failed"); + }, + ); + + it("rejects values other than true and false", () => { + const run = runFixture({ isolation: "maybe" }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain(`${PROCESS_ISOLATION_ENV} must be`); + expect(run.pidsByUri.size).toBe(0); + }); +}); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 63ac394f..514abe22 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -51,6 +51,7 @@ export type { SetupInstrumentsResponse, } from "./generated/openapi"; export { getV8Flags, tryIntrospect } from "./introspection"; +export * from "./isolation"; export { getUnsupportedNodeVersionWarning, SUPPORTED_NODE_MAJORS, diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts new file mode 100644 index 00000000..0a8e013a --- /dev/null +++ b/packages/core/src/isolation.ts @@ -0,0 +1,182 @@ +/** + * Process isolation: every benchmark runs in a fresh copy of the benchmark + * process, so the allocator state, JIT state, and handles that other + * benchmarks or setup code leave behind can't change its measurement. + */ +import { spawnSync, type SpawnSyncReturns } from "child_process"; +import { constants } from "os"; + +/** Isolation is on unless this is `0` or `false`. */ +export const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +/** Set on each child process: the URI of the only benchmark it runs. */ +export const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; +/** + * JSON array `[executable, ...args]` of the command that runs the benchmarks + * again. Integrations whose benchmarks run in a worker process (vitest) set it + * from the main process, because re-executing the worker alone can't work. + */ +export const ISOLATION_COMMAND_ENV = "CODSPEED_ISOLATION_COMMAND"; + +/** Exit code for a child that ended with neither an exit code nor a signal. */ +const GENERIC_FAILURE_EXIT_CODE = 1; +/** Shells report a child killed by signal N as exit code 128 + N. */ +const SIGNAL_EXIT_CODE_OFFSET = 128; + +/** How an integration runs each benchmark. */ +export type IsolationMode = + /** All benchmarks share this process and its state. */ + | { kind: "SharedProcess" } + /** Every benchmark runs in its own child process. */ + | { kind: "ProcessPerBenchmark" } + /** A child spawned in `ProcessPerBenchmark` mode, restricted to `uri`. */ + | { kind: "IsolatedBenchmark"; uri: string }; + +export class Isolation { + private static instance: Isolation | undefined; + private static readonly sharedInstance = new Isolation({ + kind: "SharedProcess", + }); + + private constructor(readonly mode: IsolationMode) {} + + /** + * The isolation of runs where no instrument measures anything (the hooks are + * not active), so there is nothing to isolate. + */ + static shared(): Isolation { + return Isolation.sharedInstance; + } + + /** The isolation of this process, read once from the environment. */ + static current(): Isolation { + Isolation.instance ??= new Isolation(modeFromEnv()); + return Isolation.instance; + } + + /** The only benchmark this process may run. It overrides user filters. */ + onlyBenchmark(): string | undefined { + switch (this.mode.kind) { + case "IsolatedBenchmark": + return this.mode.uri; + case "SharedProcess": + case "ProcessPerBenchmark": + return undefined; + } + } + + /** Whether this process must run the benchmark `uri`. */ + shouldRun(uri: string): boolean { + const only = this.onlyBenchmark(); + return only === undefined || only === uri; + } + + /** Whether this process is a child that runs one benchmark for a parent. */ + isChild(): boolean { + return this.onlyBenchmark() !== undefined; + } + + /** + * Runs the benchmark `uri`. `bench` must contain the whole measurement, + * from `startBenchmark` to `setExecutedBenchmark`. + */ + async run(uri: string, bench: () => Promise | void): Promise { + switch (this.mode.kind) { + case "SharedProcess": + case "IsolatedBenchmark": + await bench(); + return; + case "ProcessPerBenchmark": + runInChildProcess(uri); + return; + } + } + + /** Synchronous variant of {@link Isolation.run}. */ + runSync(uri: string, bench: () => void): void { + switch (this.mode.kind) { + case "SharedProcess": + case "IsolatedBenchmark": + bench(); + return; + case "ProcessPerBenchmark": + runInChildProcess(uri); + return; + } + } +} + +function modeFromEnv(): IsolationMode { + const uri = process.env[ISOLATED_BENCHMARK_ENV]; + if (uri !== undefined) { + return { kind: "IsolatedBenchmark", uri }; + } + + const value = process.env[PROCESS_ISOLATION_ENV]; + switch (value) { + case undefined: + case "1": + case "true": + return { kind: "ProcessPerBenchmark" }; + case "0": + case "false": + return { kind: "SharedProcess" }; + default: + throw new Error( + `${PROCESS_ISOLATION_ENV} must be \`true\` or \`false\`, got \`${value}\``, + ); + } +} + +/** + * Records the command of this process as the one that children run. Call it + * from the main process of integrations that measure in worker processes. It + * overwrites a command inherited from an outer process, which is not ours. + */ +export function recordIsolationCommand(): void { + process.env[ISOLATION_COMMAND_ENV] = JSON.stringify(currentCommand()); +} + +function currentCommand(): string[] { + // execArgv keeps the V8 flags (e.g. --perf-prof) that the runner set. + return [process.execPath, ...process.execArgv, ...process.argv.slice(1)]; +} + +function isolationCommand(): string[] { + const recorded = process.env[ISOLATION_COMMAND_ENV]; + if (recorded === undefined) { + return currentCommand(); + } + return JSON.parse(recorded) as string[]; +} + +/** + * Runs the benchmark command again, limited to `uri`, and exits with the + * child's code if it fails. Re-exec, not fork(): Node can't fork, and a fork + * would only copy the calling thread. + */ +function runInChildProcess(uri: string): void { + const [executable, ...args] = isolationCommand(); + // spawnSync blocks the event loop, so stdout written before this call is + // already flushed and output stays in order. + const result = spawnSync(executable, args, { + stdio: "inherit", + env: { ...process.env, [ISOLATED_BENCHMARK_ENV]: uri }, + }); + if (result.error !== undefined) { + throw new Error( + `[CodSpeed] failed to start the process for ${uri}: ${result.error.message}`, + ); + } + if (result.status === 0) { + return; + } + process.exit(exitCode(result)); +} + +/** Exit code of a failed child, in the shell convention. */ +function exitCode(result: SpawnSyncReturns): number { + if (result.signal !== null) { + return SIGNAL_EXIT_CODE_OFFSET + constants.signals[result.signal]; + } + return result.status ?? GENERIC_FAILURE_EXIT_CODE; +} diff --git a/packages/core/tests/fixtures/isolation.cjs b/packages/core/tests/fixtures/isolation.cjs new file mode 100644 index 00000000..46205d3b --- /dev/null +++ b/packages/core/tests/fixtures/isolation.cjs @@ -0,0 +1,27 @@ +// Run by `isolation.integ.test.ts`. Runs two benchmarks through `Isolation`. +const { Isolation } = require("../../dist/index.cjs.js"); + +const isolation = Isolation.current(); + +function bench(uri) { + console.log(`ran ${process.pid} ${uri} ${JSON.stringify(process.execArgv)}`); + switch (process.env.ISOLATION_FIXTURE_ACTION) { + case "exit": + process.exit(3); + case "kill": + process.kill(process.pid, "SIGKILL"); + } +} + +async function main() { + for (const uri of ["a", "b"]) { + if (!isolation.shouldRun(uri)) continue; + if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { + isolation.runSync(uri, () => bench(uri)); + } else { + await isolation.run(uri, async () => bench(uri)); + } + } +} + +main(); diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts new file mode 100644 index 00000000..8649d8c5 --- /dev/null +++ b/packages/core/tests/isolation.integ.test.ts @@ -0,0 +1,162 @@ +import { spawnSync } from "child_process"; +import { constants } from "os"; +import { join } from "path"; +import { + ISOLATED_BENCHMARK_ENV, + Isolation, + PROCESS_ISOLATION_ENV, +} from "../src/isolation"; + +const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const SIGNAL_EXIT_CODE_OFFSET = 128; +const CHILD_EXIT_CODE = 3; +const RAN_LINE = /^ran (\d+) (\S+) (.+)$/; +/** Passed to Node to check that children keep the flags of their parent. */ +const NODE_FLAG = "--no-warnings"; + +/** `Isolation.current()` reads the environment once, so load a fresh copy. */ +function currentIsolation(env: Record): Isolation { + const saved = Object.fromEntries( + Object.keys(env).map((key) => [key, process.env[key]]), + ); + setEnv(env); + try { + let isolation: Isolation | undefined; + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + isolation = require("../src/isolation").Isolation.current(); + }); + return isolation as unknown as Isolation; + } finally { + setEnv(saved); + } +} + +function setEnv(env: Record): void { + for (const [key, value] of Object.entries(env)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } +} + +describe("Isolation.current", () => { + it.each([undefined, "1", "true"])( + "runs one process per benchmark (isolation=%s)", + (value) => { + const isolation = currentIsolation({ + [PROCESS_ISOLATION_ENV]: value, + [ISOLATED_BENCHMARK_ENV]: undefined, + }); + + expect(isolation.mode).toEqual({ kind: "ProcessPerBenchmark" }); + expect(isolation.onlyBenchmark()).toBeUndefined(); + expect(isolation.isChild()).toBe(false); + expect(isolation.shouldRun("any")).toBe(true); + }, + ); + + it.each(["0", "false"])("shares one process (isolation=%s)", (value) => { + const isolation = currentIsolation({ + [PROCESS_ISOLATION_ENV]: value, + [ISOLATED_BENCHMARK_ENV]: undefined, + }); + + expect(isolation.mode).toEqual({ kind: "SharedProcess" }); + expect(isolation.isChild()).toBe(false); + expect(isolation.shouldRun("any")).toBe(true); + }); + + it("restricts a child to its benchmark, whatever the isolation setting", () => { + const isolation = currentIsolation({ + [PROCESS_ISOLATION_ENV]: "false", + [ISOLATED_BENCHMARK_ENV]: "file::bench", + }); + + expect(isolation.mode).toEqual({ + kind: "IsolatedBenchmark", + uri: "file::bench", + }); + expect(isolation.onlyBenchmark()).toBe("file::bench"); + expect(isolation.isChild()).toBe(true); + expect(isolation.shouldRun("file::bench")).toBe(true); + // Exact match only: a prefix is another benchmark. + expect(isolation.shouldRun("file::bench 2")).toBe(false); + expect(isolation.shouldRun("file")).toBe(false); + }); + + it("rejects any other value", () => { + expect(() => + currentIsolation({ + [PROCESS_ISOLATION_ENV]: "maybe", + [ISOLATED_BENCHMARK_ENV]: undefined, + }), + ).toThrow(`${PROCESS_ISOLATION_ENV} must be \`true\` or \`false\``); + }); + + it("never spawns when no instrument measures", () => { + expect(Isolation.shared().mode).toEqual({ kind: "SharedProcess" }); + }); +}); + +interface FixtureOptions { + action?: "exit" | "kill"; + sync?: boolean; +} + +function runFixture({ action, sync }: FixtureOptions = {}) { + const env: NodeJS.ProcessEnv = { + ...process.env, + ISOLATION_FIXTURE_ACTION: action, + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[ISOLATED_BENCHMARK_ENV]; + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + const result = spawnSync(process.execPath, [NODE_FLAG, FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + const ran = result.stdout + .split("\n") + .map((line) => RAN_LINE.exec(line)) + .filter((match): match is RegExpExecArray => match !== null); + return { status: result.status, ran }; +} + +describe.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, +])("Isolation child process ($name run)", ({ sync }) => { + it("runs each benchmark in a fresh process with the flags of its parent", () => { + const { status, ran } = runFixture({ sync }); + + expect(status).toBe(0); + expect(ran.map((match) => match[2])).toEqual(["a", "b"]); + expect(new Set(ran.map((match) => match[1])).size).toBe(2); + for (const match of ran) { + expect(JSON.parse(match[3])).toEqual([NODE_FLAG]); + } + }); + + it("exits with the exit code of a failing child", () => { + const { status, ran } = runFixture({ sync, action: "exit" }); + + expect(status).toBe(CHILD_EXIT_CODE); + // The parent stops at the first failure. + expect(ran.map((match) => match[2])).toEqual(["a"]); + }); + + it("exits with 128 + N when a signal N killed the child", () => { + const { status, ran } = runFixture({ sync, action: "kill" }); + + expect(status).toBe(SIGNAL_EXIT_CODE_OFFSET + constants.signals.SIGKILL); + expect(ran.map((match) => match[2])).toEqual(["a"]); + }); +}); diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 65bd6183..750925e5 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -1,6 +1,7 @@ import { getCodspeedRunnerMode, InstrumentHooks, + Isolation, mongoMeasurement, optimizeFunction, optimizeFunctionSync, @@ -14,7 +15,11 @@ export function setupCodspeedAnalysisBench( bench: Bench, rootCallingFile: string, ): void { - const runner = new AnalysisBenchRunner(bench, rootCallingFile); + // Without instrumentation nothing is measured, so there is nothing to isolate. + const isolation = InstrumentHooks.isInstrumented() + ? Isolation.current() + : Isolation.shared(); + const runner = new AnalysisBenchRunner(bench, rootCallingFile, isolation); runner.setupBenchMethods(); } diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 0fb811c5..14828bac 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,4 +1,4 @@ -import { setupCore, teardownCore } from "@codspeed/core"; +import { Isolation, setupCore, teardownCore } from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; @@ -8,10 +8,19 @@ declare const __VERSION__: string; export abstract class BaseBenchRunner { protected bench: Bench; protected rootCallingFile: string; - - constructor(bench: Bench, rootCallingFile: string) { + /** Runners that measure pass the isolation of this process; walltime shares it. */ + private readonly isolation: Isolation; + private readonly isChild: boolean; + + constructor( + bench: Bench, + rootCallingFile: string, + isolation: Isolation = Isolation.shared(), + ) { this.bench = bench; this.rootCallingFile = rootCallingFile; + this.isolation = isolation; + this.isChild = isolation.isChild(); } private setupBenchRun(): void { @@ -20,6 +29,10 @@ export abstract class BaseBenchRunner { } private logStart(): void { + // The parent already printed the banner, children stay quiet. + if (this.isChild) { + return; + } console.log( `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (${this.getModeName()})`, ); @@ -45,7 +58,12 @@ export abstract class BaseBenchRunner { protected finalizeBenchRun(): Task[] { teardownCore(); - console.log(`[CodSpeed] Done running ${this.bench.tasks.length} benches.`); + // The parent already printed the summary line, children stay quiet. + if (!this.isChild) { + console.log( + `[CodSpeed] Done running ${this.bench.tasks.length} benches.`, + ); + } return this.bench.tasks; } @@ -57,25 +75,40 @@ export abstract class BaseBenchRunner { public setupBenchMethods(): void { this.bench.run = async () => { + const tasks = this.selectTasks(); + // A child that owns none of this bench's tasks must not touch the + // instruments, the bench that owns its task does. + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); - await this.runTaskAsync(task, uri); + for (const { task, uri } of tasks) { + await this.isolation.run(uri, () => this.runTaskAsync(task, uri)); } return this.finalizeAsyncRun(); }; this.bench.runSync = () => { + const tasks = this.selectTasks(); + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); - this.runTaskSync(task, uri); + for (const { task, uri } of tasks) { + this.isolation.runSync(uri, () => this.runTaskSync(task, uri)); } return this.finalizeSyncRun(); }; } + + /** The tasks this process measures: all of them, or the isolated one. */ + private selectTasks(): { task: Task; uri: string }[] { + return this.bench.tasks + .map((task) => ({ task, uri: this.getTaskUri(task) })) + .filter(({ uri }) => this.isolation.shouldRun(uri)); + } } diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs new file mode 100644 index 00000000..ec2424da --- /dev/null +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -0,0 +1,46 @@ +// Run by `isolation.integ.test.ts`. Prints the PID that measures each benchmark. +import { InstrumentHooks } from "@codspeed/core"; +import { Bench } from "tinybench"; +import { withCodSpeed } from "../../dist/index.es5.js"; + +// There is no instrument hook runtime outside the CodSpeed runner, so pretend +// the hooks are active. The child processes run this file again, so they get +// the same override. +InstrumentHooks.isInstrumented = () => true; + +// The plugin reports every measured benchmark here, with the PID of the +// process that measured it. +const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark; +InstrumentHooks.setExecutedBenchmark = (pid, uri) => { + setExecutedBenchmark(pid, uri); + console.log(`executed ${pid} ${uri}`); +}; + +// tinybench calls each function once when it is added, so a failing benchmark +// may only throw once the measurement started. +let measuring = false; +const startBenchmark = InstrumentHooks.startBenchmark; +InstrumentHooks.startBenchmark = () => { + measuring = true; + startBenchmark(); +}; + +const bench = withCodSpeed(new Bench({ time: 1, warmupTime: 0 })); + +for (const name of ["first", "second", "third"]) { + bench.add(name, () => {}); +} + +if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + bench.add("failing", () => { + if (measuring) { + throw new Error("benchmark failed"); + } + }); +} + +if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { + bench.runSync(); +} else { + await bench.run(); +} diff --git a/packages/tinybench-plugin/tests/index.integ.test.ts b/packages/tinybench-plugin/tests/index.integ.test.ts index 5e71eeb2..6de91c6e 100644 --- a/packages/tinybench-plugin/tests/index.integ.test.ts +++ b/packages/tinybench-plugin/tests/index.integ.test.ts @@ -41,6 +41,9 @@ vi.mock("@codspeed/core", async (importOriginal) => { beforeEach(() => { process.env.CODSPEED_ENV = "true"; process.env.CODSPEED_RUNNER_MODE = "instrumentation"; + // These tests drive the plugin in this process against a mocked core, so + // there is no benchmark process to spawn. + process.env.CODSPEED_PROCESS_ISOLATION = "false"; vi.clearAllMocks(); }); diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts new file mode 100644 index 00000000..27970175 --- /dev/null +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -0,0 +1,122 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawnSync } from "child_process"; +import { fileURLToPath } from "url"; +import { describe, expect, it } from "vitest"; + +const FIXTURE = fileURLToPath( + new URL("./fixtures/isolation.mjs", import.meta.url), +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const BENCHMARK_COUNT = 3; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const EXECUTED_LINE = /^executed (\d+) (.+)$/; + +interface FixtureRun { + status: number | null; + stderr: string; + /** PID that measured each benchmark, in the order they were measured. */ + pidsByUri: Map; + /** PID of the process the test started. */ + parentPid: number; +} + +interface FixtureOptions { + /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ + isolation?: string; + failing?: boolean; + sync?: boolean; +} + +function runFixture({ + isolation, + failing, + sync, +}: FixtureOptions = {}): FixtureRun { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + }; + delete env[PROCESS_ISOLATION_ENV]; + if (isolation !== undefined) { + env[PROCESS_ISOLATION_ENV] = isolation; + } + if (failing) { + env.ISOLATION_FIXTURE_FAIL = "1"; + } + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + + // The runner starts Node with these flags, and children must keep them. + const result = spawnSync(process.execPath, [...getV8Flags(), FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + + const pidsByUri = new Map(); + for (const line of result.stdout.split("\n")) { + const match = EXECUTED_LINE.exec(line); + if (match) { + pidsByUri.set(match[2], Number(match[1])); + } + } + return { + status: result.status, + stderr: result.stderr, + pidsByUri, + parentPid: result.pid, + }; +} + +describe.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, +])("process isolation ($name run)", ({ sync }) => { + it.each([undefined, "1", "true"])( + "measures every benchmark in its own process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, sync }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + // The parent only spawns children, it never measures. + expect(pids).not.toContain(run.parentPid); + }, + ); + + it.each(["0", "false"])( + "measures all benchmarks in one process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, sync }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values())).toEqual(new Set([run.parentPid])); + }, + ); + + it.each(["true", "false"])( + "fails the run when a benchmark throws (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, failing: true, sync }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain("benchmark failed"); + }, + ); +}); + +describe("process isolation setting", () => { + it("rejects values other than true and false", () => { + const run = runFixture({ isolation: "maybe" }); + + expect(run.status).not.toBe(0); + expect(run.stderr).toContain(`${PROCESS_ISOLATION_ENV} must be`); + expect(run.pidsByUri.size).toBe(0); + }); +}); diff --git a/packages/vitest-plugin/src/__tests__/instrumented.test.ts b/packages/vitest-plugin/src/__tests__/instrumented.test.ts index ec1a1fd9..2e7f54c6 100644 --- a/packages/vitest-plugin/src/__tests__/instrumented.test.ts +++ b/packages/vitest-plugin/src/__tests__/instrumented.test.ts @@ -6,6 +6,8 @@ import { getBenchFn, getBenchOptions } from "../legacy/compat"; const coreMocks = vi.hoisted(() => { return { InstrumentHooks: { + // A plain function: the mocks above are reset between tests. + isInstrumented: () => true, startBenchmark: vi.fn(), stopBenchmark: vi.fn(), setExecutedBenchmark: vi.fn(), @@ -21,6 +23,10 @@ const coreMocks = vi.hoisted(() => { global.eval = vi.fn(); +// The runner is driven in this process against a mocked core, so there is no +// benchmark process to spawn. +process.env.CODSPEED_PROCESS_ISOLATION = "false"; + vi.mock("@codspeed/core", async (importOriginal) => { const mod = await importOriginal(); return { ...mod, ...coreMocks }; diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts new file mode 100644 index 00000000..6683ccf1 --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -0,0 +1,113 @@ +import { spawnSync } from "child_process"; +import { mkdtempSync, readFileSync, rmSync } from "fs"; +import { createRequire } from "module"; +import { tmpdir } from "os"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const FIXTURE_DIR = fileURLToPath( + new URL("../../tests/fixtures/isolation", import.meta.url), +); +const PRELOAD = join(FIXTURE_DIR, "stub.mjs"); +const VITEST_BIN = join( + dirname(createRequire(import.meta.url).resolve("vitest/package.json")), + "vitest.mjs", +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const BENCHMARK_COUNT = 3; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 120_000; +const EXECUTED_LINE = /^executed (\d+) (.+)$/; + +let logDir: string; +let logFile: string; + +beforeEach(() => { + logDir = mkdtempSync(join(tmpdir(), "codspeed-isolation-")); + logFile = join(logDir, "executed.log"); +}); + +afterEach(() => { + rmSync(logDir, { recursive: true, force: true }); +}); + +interface FixtureOptions { + /** Value of CODSPEED_PROCESS_ISOLATION, unset when undefined. */ + isolation?: string; + failing?: boolean; +} + +/** Runs `vitest bench` on the fixture project and returns what it measured. */ +function runFixture({ isolation, failing }: FixtureOptions = {}) { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + ISOLATION_LOG_FILE: logFile, + NODE_OPTIONS: [process.env.NODE_OPTIONS, `--import ${PRELOAD}`] + .filter(Boolean) + .join(" "), + }; + delete env[PROCESS_ISOLATION_ENV]; + if (isolation !== undefined) { + env[PROCESS_ISOLATION_ENV] = isolation; + } + if (failing) { + env.ISOLATION_FIXTURE_FAIL = "1"; + } + + const result = spawnSync(process.execPath, [VITEST_BIN, "bench", "--run"], { + cwd: FIXTURE_DIR, + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + + const pidsByUri = new Map(); + for (const line of readFileSync(logFile, "utf8").split("\n")) { + const match = EXECUTED_LINE.exec(line); + if (match) { + pidsByUri.set(match[2], Number(match[1])); + } + } + return { status: result.status, pidsByUri, parentPid: result.pid }; +} + +describe("process isolation", () => { + it.each([undefined, "1", "true"])( + "measures every benchmark in its own process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + const pids = new Set(run.pidsByUri.values()); + expect(pids.size).toBe(BENCHMARK_COUNT); + expect(pids).not.toContain(run.parentPid); + }, + RUN_TIMEOUT_MS, + ); + + it.each(["0", "false"])( + "measures all benchmarks in one process (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation }); + + expect(run.status).toBe(0); + expect(run.pidsByUri.size).toBe(BENCHMARK_COUNT); + expect(new Set(run.pidsByUri.values()).size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it.each(["true", "false"])( + "fails the run when a benchmark throws (isolation=%s)", + (isolation) => { + const run = runFixture({ isolation, failing: true }); + + expect(run.status).not.toBe(0); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index cb3af1ae..ae81cf3b 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -3,6 +3,7 @@ import { getV8Flags, InstrumentHooks, mongoMeasurement, + recordIsolationCommand, SetupInstrumentsRequestBody, SetupInstrumentsResponse, } from "@codspeed/core"; @@ -38,6 +39,10 @@ export default function codspeedPlugin(): Plugin { }, enforce: "post", config(userConfig): ViteUserConfig { + // Vitest measures in forked workers, whose argv is not the command that + // starts the benchmarks. Record the command of this main process so the + // workers can run it again for each benchmark. Workers inherit the env. + recordIsolationCommand(); return { test: { pool: "forks", diff --git a/packages/vitest-plugin/src/legacy/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts index f71f0c0a..7dc9f7d3 100644 --- a/packages/vitest-plugin/src/legacy/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -1,5 +1,6 @@ import { InstrumentHooks, + Isolation, logDebug, mongoMeasurement, optimizeFunction, @@ -41,44 +42,50 @@ async function runAnalysisBench( suite: RunnerTestSuite, currentSuiteName: string, tinybenchModule: Tinybench, + isolation: Isolation, ) { const uri = `${currentSuiteName}::${benchmark.name}`; - const fn = getBenchFn(benchmark); - - // Constructing a Bench applies tinybench's no-op defaults for the setup and - // teardown hooks and gives them the Task they expect. The bench itself is - // never run: this runner drives the benchmark function directly. - const bench = new tinybenchModule.Bench(getBenchOptions(benchmark)); - const task = new tinybenchModule.Task(bench, benchmark.name, fn); - - await bench.setup(task, "warmup"); - await optimizeFunction(async () => { + if (!isolation.shouldRun(uri)) return; + + await isolation.run(uri, async () => { + const fn = getBenchFn(benchmark); + + // Constructing a Bench applies tinybench's no-op defaults for the setup and + // teardown hooks and gives them the Task they expect. The bench itself is + // never run: this runner drives the benchmark function directly. + const bench = new tinybenchModule.Bench(getBenchOptions(benchmark)); + const task = new tinybenchModule.Task(bench, benchmark.name, fn); + + await bench.setup(task, "warmup"); + await optimizeFunction(async () => { + await callSuiteHook(suite, benchmark, "beforeEach"); + await fn(); + await callSuiteHook(suite, benchmark, "afterEach"); + }); + await bench.teardown(task, "warmup"); + + await bench.setup(task, "run"); await callSuiteHook(suite, benchmark, "beforeEach"); - await fn(); + await mongoMeasurement.start(uri); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await mongoMeasurement.stop(uri); await callSuiteHook(suite, benchmark, "afterEach"); + await bench.teardown(task, "run"); + + logCodSpeed(`${uri} done`); }); - await bench.teardown(task, "warmup"); - - await bench.setup(task, "run"); - await callSuiteHook(suite, benchmark, "beforeEach"); - await mongoMeasurement.start(uri); - global.gc?.(); - await wrapWithRootFrame(async () => { - InstrumentHooks.startBenchmark(); - await fn(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - await callSuiteHook(suite, benchmark, "afterEach"); - await bench.teardown(task, "run"); - - logCodSpeed(`${uri} done`); } async function runAnalysisBenchmarkSuite( suite: RunnerTestSuite, tinybenchModule: Tinybench, + isolation: Isolation, parentSuiteName?: string, ) { const currentSuiteName = parentSuiteName @@ -91,9 +98,20 @@ async function runAnalysisBenchmarkSuite( if (task.mode !== "run") continue; if (task.type === "suite") { - await runAnalysisBenchmarkSuite(task, tinybenchModule, currentSuiteName); + await runAnalysisBenchmarkSuite( + task, + tinybenchModule, + isolation, + currentSuiteName, + ); } else if (isVitestTaskBenchmark(task)) { - await runAnalysisBench(task, suite, currentSuiteName, tinybenchModule); + await runAnalysisBench( + task, + suite, + currentSuiteName, + tinybenchModule, + isolation, + ); } } @@ -107,9 +125,24 @@ export class AnalysisRunner extends NodeBenchmarkRunner { patchRootSuiteWithFullFilePath(suite); - logCodSpeed(`running suite ${suite.name}`); - await runAnalysisBenchmarkSuite(suite, await this.importTinybench()); - logCodSpeed(`running suite ${suite.name} done`); + // Without instrumentation nothing is measured, so there is nothing to isolate. + const isolation = InstrumentHooks.isInstrumented() + ? Isolation.current() + : Isolation.shared(); + // The parent already printed the suite lines, children stay quiet. + const isChild = isolation.isChild(); + + if (!isChild) { + logCodSpeed(`running suite ${suite.name}`); + } + await runAnalysisBenchmarkSuite( + suite, + await this.importTinybench(), + isolation, + ); + if (!isChild) { + logCodSpeed(`running suite ${suite.name} done`); + } teardownCore(); } diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index ef38fe0e..c54d2017 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -2,6 +2,7 @@ import { getGitDir, getInstrumentMode, InstrumentHooks, + Isolation, optimizeFunction, optimizeFunctionSync, setupCore, @@ -155,12 +156,21 @@ async function runAnalysis( registrations: BenchRegistrationInput[], getUri: (name: string) => string, ): Promise { - const label = InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; + const instrumented = InstrumentHooks.isInstrumented(); + const label = instrumented ? "Measured" : "Checked"; + // Without instrumentation nothing is measured, so there is nothing to isolate. + const isolation = instrumented ? Isolation.current() : Isolation.shared(); const results: BenchResult[] = []; for (const { name, fn, fnOpts } of registrations) { const uri = getUri(name); - await runAnalysisTask(fn, fnOpts, uri); - console.log(`[CodSpeed] ${label} ${uri}`); + // An isolated child still reports every benchmark to Vitest, but it only + // measures its own. + if (isolation.shouldRun(uri)) { + await isolation.run(uri, async () => { + await runAnalysisTask(fn, fnOpts, uri); + console.log(`[CodSpeed] ${label} ${uri}`); + }); + } results.push(analysisResult(name)); } return results; diff --git a/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts new file mode 100644 index 00000000..c803dcbc --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts @@ -0,0 +1,19 @@ +import { describe, test } from "vitest"; + +describe("isolation", () => { + test("benchmarks", async ({ bench }) => { + const benchmarks = [ + bench("first", () => {}), + bench("second", () => {}), + bench("third", () => {}), + ]; + if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + benchmarks.push( + bench("failing", () => { + throw new Error("benchmark failed"); + }), + ); + } + await bench.compare(...benchmarks); + }); +}); diff --git a/packages/vitest-plugin/tests/fixtures/isolation/stub.mjs b/packages/vitest-plugin/tests/fixtures/isolation/stub.mjs new file mode 100644 index 00000000..bdfd51b6 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/stub.mjs @@ -0,0 +1,16 @@ +// Preloaded with `node --import` into the Vitest process, its workers, and the +// Vitest runs that process isolation starts. +import { InstrumentHooks } from "@codspeed/core"; +import { appendFileSync } from "fs"; + +// There is no instrument hook runtime outside the CodSpeed runner, so pretend +// the hooks are active. +InstrumentHooks.isInstrumented = () => true; + +// Vitest swallows what workers print, so the PID that measures each benchmark +// goes to a file. +const setExecutedBenchmark = InstrumentHooks.setExecutedBenchmark; +InstrumentHooks.setExecutedBenchmark = (pid, uri) => { + setExecutedBenchmark(pid, uri); + appendFileSync(process.env.ISOLATION_LOG_FILE, `executed ${pid} ${uri}\n`); +}; diff --git a/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts new file mode 100644 index 00000000..df313ac4 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; +import codspeedPlugin from "../../../dist/index.mjs"; + +export default defineConfig({ + plugins: [codspeedPlugin()], + define: { + __VERSION__: JSON.stringify("1.0.0"), + }, + test: { + include: ["**/*.bench.ts"], + }, +});