From c86f5b195cd8d552ed2ee7473da48fe5fd446a01 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:36 +0200 Subject: [PATCH 1/3] feat(core): add per-benchmark process isolation Benchmarks of one binary share a process, so the allocator state and threads one benchmark leaves behind (raised mmap threshold, fragmented bins, retained caches) change the measurement of every benchmark after it. Adding or changing one benchmark can then shift unrelated results. Add `codspeed::Isolation`. It is on by default and turned off with `CODSPEED_PROCESS_ISOLATION=false` (or `0`); any other value than true/1/false/0 is an error. `Isolation::run` starts the benchmark executable again with the original arguments and `CODSPEED_ISOLATED_BENCHMARK` set to one benchmark URI, so each benchmark gets a fresh process. A failing child's exit code is propagated (128 + N for signal N). Re-exec instead of fork(): fork only copies the calling thread, so threads the binary started before the harness would be missing in the child and work sent to them would hang. Linux uses /proc/self/exe, macOS _NSGetExecutablePath; Windows is not supported. --- core/CMakeLists.txt | 1 + core/include/isolation.h | 60 +++++++++++ core/src/isolation.cpp | 212 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 273 insertions(+) create mode 100644 core/include/isolation.h create mode 100644 core/src/isolation.cpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index f0d5eb2..83a2d37 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -85,6 +85,7 @@ endif() add_library( codspeed src/codspeed.cpp + src/isolation.cpp src/measurement.cpp src/walltime.cpp src/uri.cpp diff --git a/core/include/isolation.h b/core/include/isolation.h new file mode 100644 index 0000000..8924c56 --- /dev/null +++ b/core/include/isolation.h @@ -0,0 +1,60 @@ +#ifndef CODSPEED_ISOLATION_H +#define CODSPEED_ISOLATION_H + +#include +#include + +namespace codspeed { + +// Isolation is on unless this is `0` or `false`. +extern const char* const PROCESS_ISOLATION_ENV; +// Set on each child process: the URI of the only benchmark it runs. +extern const char* const ISOLATED_BENCHMARK_ENV; + +// How the harness runs each benchmark. Every benchmark runs in a fresh copy of +// the benchmark executable by default, so the allocator state and threads that +// other benchmarks or setup code leave behind can't change its measurement. +class Isolation { + public: + enum class Kind { + // All benchmarks share this process and its allocator state. + SharedProcess, + // Every benchmark runs in its own child process. + ProcessPerBenchmark, + // A child spawned by `ProcessPerBenchmark`, restricted to one URI. + IsolatedBenchmark, + }; + + // Stores the original command line. Call it before the arguments get + // parsed, because parsing removes the flags it consumed. The child process + // is started with the same arguments. + static void set_argv(int argc, const char* const* argv); + + // The isolation of this process, read once from the environment. + static const Isolation& current(); + + Kind kind() const { return kind_; } + + // The only benchmark this process may run, or null. It overrides CLI + // filters. + const std::string* only_benchmark() const; + + // Runs the benchmark `uri`. `bench` must contain the whole measurement, + // from `start_benchmark` to `end_benchmark`. + void run(const std::string& uri, const std::function& bench) const; + + // Runs this executable again with the same arguments, limited to `uri`, and + // exits with the child's code if it fails. + static void run_in_child_process(const std::string& uri); + + private: + Isolation(Kind kind, std::string isolated_uri); + static Isolation from_env(); + + Kind kind_; + std::string isolated_uri_; +}; + +} // namespace codspeed + +#endif // CODSPEED_ISOLATION_H diff --git a/core/src/isolation.cpp b/core/src/isolation.cpp new file mode 100644 index 0000000..fcacb7a --- /dev/null +++ b/core/src/isolation.cpp @@ -0,0 +1,212 @@ +#include "isolation.h" + +#include +#include +#include +#include +#include +#include + +#include "utils.h" + +#ifndef _WIN32 +#include +#include +#include + +#include +#ifdef __APPLE__ +#include +#endif + +extern char** environ; +#endif + +namespace codspeed { + +const char* const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const char* const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; + +namespace { + +// Exit code for a child that ended with neither an exit code nor a signal. +constexpr int GENERIC_FAILURE_EXIT_CODE = 1; +// Shells report a child killed by signal N as exit code 128 + N. +constexpr int SIGNAL_EXIT_CODE_OFFSET = 128; + +#ifndef _WIN32 +// Path of the running executable. `/proc/self/exe` keeps working when the +// benchmark changed its working directory or `argv[0]` is a bare name. +std::string current_executable() { +#if defined(__linux__) + return "/proc/self/exe"; +#elif defined(__APPLE__) + uint32_t size = 0; + _NSGetExecutablePath(nullptr, &size); + std::vector buffer(size); + if (_NSGetExecutablePath(buffer.data(), &size) == 0) { + return std::string(buffer.data()); + } + return std::string(); +#else + return std::string(); +#endif +} + +// The environment of this process plus the isolated benchmark URI. +std::vector child_environment(const std::string& uri) { + std::vector env; + const std::string isolated_prefix = std::string(ISOLATED_BENCHMARK_ENV) + "="; + for (char** entry = environ; entry != nullptr && *entry != nullptr; ++entry) { + if (std::strncmp(*entry, isolated_prefix.c_str(), isolated_prefix.size()) == + 0) { + continue; + } + env.emplace_back(*entry); + } + env.push_back(isolated_prefix + uri); + return env; +} + +// Exit code of a failed child, in the shell convention. +int exit_code(int wait_status) { + if (WIFSIGNALED(wait_status)) { + return SIGNAL_EXIT_CODE_OFFSET + WTERMSIG(wait_status); + } + if (WIFEXITED(wait_status)) { + return WEXITSTATUS(wait_status); + } + return GENERIC_FAILURE_EXIT_CODE; +} +#endif + +std::vector& original_argv() { + static std::vector argv; + return argv; +} + +} // namespace + +void Isolation::set_argv(int argc, const char* const* argv) { + auto& stored = original_argv(); + stored.clear(); + for (int i = 0; i < argc; ++i) { + stored.emplace_back(argv[i]); + } +} + +Isolation::Isolation(Kind kind, std::string isolated_uri) + : kind_(kind), isolated_uri_(std::move(isolated_uri)) {} + +const Isolation& Isolation::current() { + static const Isolation instance = from_env(); + return instance; +} + +Isolation Isolation::from_env() { + const std::string uri = safe_getenv(ISOLATED_BENCHMARK_ENV); + if (!uri.empty()) { + return Isolation(Kind::IsolatedBenchmark, uri); + } + + const std::string value = safe_getenv(PROCESS_ISOLATION_ENV); + if (value.empty() || value == "1" || value == "true") { + return Isolation(Kind::ProcessPerBenchmark, ""); + } + if (value == "0" || value == "false") { + return Isolation(Kind::SharedProcess, ""); + } + + std::cerr << PROCESS_ISOLATION_ENV << " must be `true` or `false`, got `" + << value << "`" << std::endl; + std::exit(GENERIC_FAILURE_EXIT_CODE); +} + +const std::string* Isolation::only_benchmark() const { + return kind_ == Kind::IsolatedBenchmark ? &isolated_uri_ : nullptr; +} + +void Isolation::run(const std::string& uri, + const std::function& bench) const { + switch (kind_) { + case Kind::SharedProcess: + case Kind::IsolatedBenchmark: + bench(); + return; + case Kind::ProcessPerBenchmark: + run_in_child_process(uri); + return; + } +} + +#ifdef _WIN32 +void Isolation::run_in_child_process(const std::string& uri) { + std::cerr << "process isolation is not supported on Windows, set " + << PROCESS_ISOLATION_ENV << "=false (benchmark: " << uri << ")" + << std::endl; + std::exit(GENERIC_FAILURE_EXIT_CODE); +} +#else +void Isolation::run_in_child_process(const std::string& uri) { + // Keep the parent's output before the child's. + std::fflush(stdout); + std::cout.flush(); + std::cerr.flush(); + + const std::string exe = current_executable(); + const auto& args = original_argv(); + if (exe.empty() && args.empty()) { + std::cerr << "failed to locate the benchmark executable for " << uri + << std::endl; + std::exit(GENERIC_FAILURE_EXIT_CODE); + } + + // Fall back to argv[0] where the OS gives no executable path. + const std::string& program = exe.empty() ? args[0] : exe; + + std::vector argv_storage = args; + if (argv_storage.empty()) { + argv_storage.push_back(program); + } + std::vector argv; + for (auto& arg : argv_storage) { + argv.push_back(&arg[0]); + } + argv.push_back(nullptr); + + std::vector env_storage = child_environment(uri); + std::vector envp; + for (auto& entry : env_storage) { + envp.push_back(&entry[0]); + } + envp.push_back(nullptr); + + // Re-exec instead of fork(): fork only copies the calling thread, so threads + // started before the harness (e.g. in main) would be missing in the child. + pid_t pid = 0; + const int spawn_error = posix_spawn(&pid, program.c_str(), nullptr, nullptr, + argv.data(), envp.data()); + if (spawn_error != 0) { + std::cerr << "failed to start the process for " << uri << ": " + << std::strerror(spawn_error) << std::endl; + std::exit(GENERIC_FAILURE_EXIT_CODE); + } + + int wait_status = 0; + while (waitpid(pid, &wait_status, 0) < 0) { + if (errno == EINTR) { + continue; + } + std::cerr << "failed to wait for the process of " << uri << ": " + << std::strerror(errno) << std::endl; + std::exit(GENERIC_FAILURE_EXIT_CODE); + } + + if (WIFEXITED(wait_status) && WEXITSTATUS(wait_status) == 0) { + return; + } + std::exit(exit_code(wait_status)); +} +#endif + +} // namespace codspeed From 4ed2aee733af72806ca50dfc4ad0138bc84526fc Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:54 +0200 Subject: [PATCH 2/3] feat(google_benchmark): run each benchmark in its own process In analysis builds, store the original argv in `Initialize` and route every benchmark through `Isolation::run`, one child per benchmark including its repetitions. A child only matches its own benchmark name, which overrides `--benchmark_filter`, and does not print the mode line and context again. Walltime builds are unchanged. --- google_benchmark/src/benchmark.cc | 50 ++++++++++++++++++++-- google_benchmark/src/benchmark_register.cc | 17 ++++++-- google_benchmark/src/benchmark_runner.h | 2 +- 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/google_benchmark/src/benchmark.cc b/google_benchmark/src/benchmark.cc index 610a62c..20fd4f2 100644 --- a/google_benchmark/src/benchmark.cc +++ b/google_benchmark/src/benchmark.cc @@ -20,6 +20,7 @@ #include "internal_macros.h" #include "measurement.hpp" +#include "isolation.h" #ifndef BENCHMARK_OS_WINDOWS #if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) @@ -464,9 +465,19 @@ void RunBenchmarks(const std::vector& benchmarks, // Note the file_reporter can be null. BM_CHECK(display_reporter != nullptr); +#ifdef CODSPEED_ANALYSIS + const codspeed::Isolation& isolation = codspeed::Isolation::current(); + // Process isolation children only run one benchmark. The parent already + // printed the mode and the context. + const bool is_isolated_child = isolation.only_benchmark() != nullptr; +#endif + #ifdef CODSPEED_ENABLED auto& Err = display_reporter->GetErrorStream(); - Err << "Codspeed mode: " << CODSPEED_MODE_DISPLAY << "\n"; +#ifdef CODSPEED_ANALYSIS + if (!is_isolated_child) +#endif + Err << "Codspeed mode: " << CODSPEED_MODE_DISPLAY << "\n"; #endif // CODSPEED_ENABLED bool might_have_aggregates = FLAGS_benchmark_repetitions > 1; @@ -493,8 +504,18 @@ void RunBenchmarks(const std::vector& benchmarks, std::map per_family_reports; - if (display_reporter->ReportContext(context) && - ((file_reporter == nullptr) || file_reporter->ReportContext(context))) { +#ifdef CODSPEED_ANALYSIS + const bool context_reported = + is_isolated_child || + (display_reporter->ReportContext(context) && + ((file_reporter == nullptr) || file_reporter->ReportContext(context))); +#else + const bool context_reported = + display_reporter->ReportContext(context) && + ((file_reporter == nullptr) || file_reporter->ReportContext(context)); +#endif + + if (context_reported) { FlushStreams(display_reporter); FlushStreams(file_reporter); @@ -560,10 +581,26 @@ void RunBenchmarks(const std::vector& benchmarks, #ifdef CODSPEED_WALLTIME std::vector codspeed_walltime_data; +#endif +#ifdef CODSPEED_ANALYSIS + // The child of a benchmark runs all of its repetitions, so the parent + // starts only one child per benchmark. + std::vector isolated(runners.size(), false); #endif for (size_t repetition_index : repetition_indices) { internal::BenchmarkRunner& runner = runners[repetition_index]; +#ifdef CODSPEED_ANALYSIS + if (isolation.kind() == codspeed::Isolation::Kind::ProcessPerBenchmark) { + if (!isolated[repetition_index]) { + isolated[repetition_index] = true; + isolation.run(runner.GetBenchmarkName(), + [&runner]() { runner.DoOneRepetition(); }); + } + continue; + } +#endif + #ifdef CODSPEED_WALLTIME auto* codspeed = codspeed::CodSpeed::getInstance(); if (codspeed != nullptr && runner.IsFirstRepetition()) { @@ -954,6 +991,13 @@ void PrintDefaultHelp() { } void Initialize(int* argc, char** argv, void (*HelperPrintf)()) { +#ifdef CODSPEED_ANALYSIS + // Keep the full command line: parsing below removes the benchmark flags, and + // process isolation starts the children with the original arguments. + codspeed::Isolation::set_argv(*argc, argv); + // Reject an invalid CODSPEED_PROCESS_ISOLATION before running anything. + codspeed::Isolation::current(); +#endif internal::HelperPrintf = HelperPrintf; internal::ParseCommandLineFlags(argc, argv); internal::LogLevel() = FLAGS_v; diff --git a/google_benchmark/src/benchmark_register.cc b/google_benchmark/src/benchmark_register.cc index 9904b9d..3f5c75c 100644 --- a/google_benchmark/src/benchmark_register.cc +++ b/google_benchmark/src/benchmark_register.cc @@ -13,6 +13,9 @@ // limitations under the License. #include "benchmark_register.h" +#ifdef CODSPEED_ANALYSIS +#include "isolation.h" +#endif #ifndef BENCHMARK_OS_WINDOWS #if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) @@ -172,9 +175,17 @@ bool BenchmarkFamilies::FindBenchmarks( num_threads); const auto full_name = instance.name().str(); - if (full_name.rfind(kDisabledPrefix, 0) != 0 && - ((re.Match(full_name) && !is_negative_filter) || - (!re.Match(full_name) && is_negative_filter))) { + bool matches_filter = (re.Match(full_name) && !is_negative_filter) || + (!re.Match(full_name) && is_negative_filter); +#ifdef CODSPEED_ANALYSIS + // A process isolation child runs exactly one benchmark, whatever the + // command line filter says. + if (const std::string* only = + codspeed::Isolation::current().only_benchmark()) { + matches_filter = full_name == *only; + } +#endif + if (full_name.rfind(kDisabledPrefix, 0) != 0 && matches_filter) { benchmarks->push_back(std::move(instance)); ++per_family_instance_index; diff --git a/google_benchmark/src/benchmark_runner.h b/google_benchmark/src/benchmark_runner.h index e89ba87..5dc4d53 100644 --- a/google_benchmark/src/benchmark_runner.h +++ b/google_benchmark/src/benchmark_runner.h @@ -124,7 +124,7 @@ class BenchmarkRunner { void RunWarmUp(); -#ifdef CODSPEED_WALLTIME +#if defined(CODSPEED_WALLTIME) || defined(CODSPEED_ANALYSIS) public: inline std::string GetBenchmarkName() { return b.name().str(); From b84d7832ad5dadae6ff8f27912f15bfa1aa8a925 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 21 Oct 2026 11:55:55 +0200 Subject: [PATCH 3/3] test: cover process isolation end to end Build a fixture in simulation mode and check that: - each benchmark runs in its own process by default, including one that uses a thread started in main; - CODSPEED_PROCESS_ISOLATION=false runs them all in one process; - a failing benchmark fails the run with its exit code; - the child ignores the command line filter; - an invalid setting is rejected. Register it with ctest and as a Bazel sh_test, and run it in CI. --- .github/workflows/ci.yml | 16 ++++ tests/process_isolation/BUILD.bazel | 18 ++++ tests/process_isolation/CMakeLists.txt | 46 +++++++++ tests/process_isolation/main.cpp | 124 +++++++++++++++++++++++++ tests/process_isolation/run_test.sh | 88 ++++++++++++++++++ 5 files changed, 292 insertions(+) create mode 100644 tests/process_isolation/BUILD.bazel create mode 100644 tests/process_isolation/CMakeLists.txt create mode 100644 tests/process_isolation/main.cpp create mode 100755 tests/process_isolation/run_test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af09a5b..80c1452 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,6 +102,22 @@ jobs: mode: ${{ matrix.codspeed-mode }} run: examples/google_benchmark_cmake/build/benchmark_example + process-isolation-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + submodules: "recursive" + + - name: Build and run the process isolation test + run: | + mkdir -p tests/process_isolation/build + cd tests/process_isolation/build + cmake -DCODSPEED_MODE=simulation -DCODSPEED_STRICT_WARNINGS=ON .. + make -j + ctest --output-on-failure + bazel-integration-tests: strategy: matrix: diff --git a/tests/process_isolation/BUILD.bazel b/tests/process_isolation/BUILD.bazel new file mode 100644 index 0000000..256319d --- /dev/null +++ b/tests/process_isolation/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "process_isolation_fixture", + srcs = ["main.cpp"], + copts = ["-Wall", "-Wextra", "-Werror"], + linkopts = ["-pthread"], + deps = ["@codspeed_google_benchmark_compat//:benchmark"], +) + +# Run with --@codspeed_core//:codspeed_mode=simulation (or another analysis +# mode), process isolation does not exist in the other modes. +sh_test( + name = "process_isolation_test", + srcs = ["run_test.sh"], + args = ["$(location :process_isolation_fixture)"], + data = [":process_isolation_fixture"], +) diff --git a/tests/process_isolation/CMakeLists.txt b/tests/process_isolation/CMakeLists.txt new file mode 100644 index 0000000..ed1d4fa --- /dev/null +++ b/tests/process_isolation/CMakeLists.txt @@ -0,0 +1,46 @@ +cmake_minimum_required(VERSION 3.12) +include(FetchContent) + +project(codspeed_process_isolation_test VERSION 0.0.0 LANGUAGES CXX) + +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + add_compile_options(-Wall -Wextra -Werror) +endif() + +set(CMAKE_BUILD_TYPE RelWithDebInfo) + +# Keep ctest limited to the isolation test. +set(BENCHMARK_ENABLE_TESTING + OFF + CACHE BOOL + "Enable testing of the benchmark library." + FORCE +) +set(BENCHMARK_ENABLE_GTEST_TESTS + OFF + CACHE BOOL + "Enable testing of the benchmark library." + FORCE +) + +FetchContent_Declare( + google_benchmark + SOURCE_DIR ${CMAKE_SOURCE_DIR}/../../google_benchmark +) +FetchContent_MakeAvailable(google_benchmark) + +find_package(Threads REQUIRED) +add_executable(process_isolation_fixture main.cpp) +target_link_libraries( + process_isolation_fixture + PRIVATE benchmark::benchmark Threads::Threads +) + +# Process isolation only exists in the analysis modes. +enable_testing() +add_test( + NAME process_isolation + COMMAND + ${CMAKE_CURRENT_SOURCE_DIR}/run_test.sh + $ +) diff --git a/tests/process_isolation/main.cpp b/tests/process_isolation/main.cpp new file mode 100644 index 0000000..0ea33f1 --- /dev/null +++ b/tests/process_isolation/main.cpp @@ -0,0 +1,124 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Set to make `BM_Failing` exit with `FAILING_EXIT_CODE`. +static const char* const FAIL_ENV = "FIXTURE_FAIL"; +static constexpr int FAILING_EXIT_CODE = 3; + +// Started in `main`, before any benchmark, so its thread belongs to the +// process that runs the harness. A forked child would not have it. +class MainThreadWorker { + public: + MainThreadWorker() : thread_([this] { loop(); }) {} + + ~MainThreadWorker() { + { + std::lock_guard lock(mutex_); + done_ = true; + } + wakeup_.notify_one(); + thread_.join(); + } + + int submit(int value) { + std::promise result; + { + std::lock_guard lock(mutex_); + jobs_.push([&result, value] { result.set_value(value + 1); }); + } + wakeup_.notify_one(); + return result.get_future().get(); + } + + private: + void loop() { + std::unique_lock lock(mutex_); + while (true) { + wakeup_.wait(lock, [this] { return done_ || !jobs_.empty(); }); + if (jobs_.empty()) { + return; + } + auto job = std::move(jobs_.front()); + jobs_.pop(); + lock.unlock(); + job(); + lock.lock(); + } + } + + std::mutex mutex_; + std::condition_variable wakeup_; + std::queue> jobs_; + bool done_ = false; + std::thread thread_; +}; + +static MainThreadWorker* worker = nullptr; + +// The test reads these lines to count the processes. +static void print_pid_once(const char* name) { + static std::mutex print_mutex; + std::lock_guard lock(print_mutex); + static std::string printed; + if (printed.find(name) != std::string::npos) { + return; + } + printed += name; + std::printf("PID %s %d\n", name, static_cast(getpid())); + std::fflush(stdout); +} + +static void BM_First(benchmark::State& state) { + print_pid_once("first"); + for (auto _ : state) { + benchmark::DoNotOptimize(1); + } +} +BENCHMARK(BM_First); + +static void BM_Second(benchmark::State& state) { + print_pid_once("second"); + for (auto _ : state) { + benchmark::DoNotOptimize(2); + } +} +BENCHMARK(BM_Second); + +static void BM_OnMainWorker(benchmark::State& state) { + print_pid_once("worker"); + for (auto _ : state) { + benchmark::DoNotOptimize(worker->submit(3)); + } +} +BENCHMARK(BM_OnMainWorker); + +static void BM_Failing(benchmark::State& state) { + print_pid_once("failing"); + for (auto _ : state) { + std::exit(FAILING_EXIT_CODE); + } +} + +int main(int argc, char** argv) { + MainThreadWorker main_worker; + worker = &main_worker; + + if (std::getenv(FAIL_ENV) != nullptr) { + benchmark::RegisterBenchmark("BM_Failing", BM_Failing); + } + + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + return 0; +} diff --git a/tests/process_isolation/run_test.sh b/tests/process_isolation/run_test.sh new file mode 100755 index 0000000..f739b24 --- /dev/null +++ b/tests/process_isolation/run_test.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# End to end test of process isolation: runs the fixture binary, built in an +# analysis mode, and checks the processes it used and its exit code. +set -u + +FIXTURE="$1" +EXPECTED_BENCHMARKS=(BM_First BM_Second BM_OnMainWorker) +FAILING_EXIT_CODE=3 +TIMEOUT_SECONDS=120 + +failures=0 +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} + +run_fixture() { + timeout "$TIMEOUT_SECONDS" "$FIXTURE" "$@" >"$OUTPUT_FILE" 2>"$ERROR_FILE" + return $? +} + +OUTPUT_FILE="$(mktemp)" +ERROR_FILE="$(mktemp)" +trap 'rm -f "$OUTPUT_FILE" "$ERROR_FILE"' EXIT + +distinct_pids() { + grep '^PID ' "$OUTPUT_FILE" | awk '{print $3}' | sort -u | wc -l +} + +check_all_benchmarks_ran() { + local label="$1" + for name in "${EXPECTED_BENCHMARKS[@]}"; do + grep -q "::${name}" "$ERROR_FILE" || fail "$label: no 'Measured/Checked' line for $name" + done +} + +# 1. Isolation on (default and explicit): one process per benchmark. +for setting in "" "true" "1"; do + label="isolation=${setting:-}" + if [ -z "$setting" ]; then + env -u CODSPEED_PROCESS_ISOLATION -u FIXTURE_FAIL \ + timeout "$TIMEOUT_SECONDS" "$FIXTURE" >"$OUTPUT_FILE" 2>"$ERROR_FILE" + else + CODSPEED_PROCESS_ISOLATION="$setting" run_fixture + fi + [ $? -eq 0 ] || fail "$label: expected exit code 0" + check_all_benchmarks_ran "$label" + pids=$(distinct_pids) + [ "$pids" -eq "${#EXPECTED_BENCHMARKS[@]}" ] || + fail "$label: expected ${#EXPECTED_BENCHMARKS[@]} distinct PIDs, got $pids" + # The parent prints the context once, children must stay quiet. + [ "$(grep -c 'Run on' "$ERROR_FILE")" -le 1 ] || fail "$label: context printed more than once" +done + +# 2. Isolation off: all benchmarks share one process. +for setting in "false" "0"; do + label="isolation=$setting" + CODSPEED_PROCESS_ISOLATION="$setting" run_fixture + [ $? -eq 0 ] || fail "$label: expected exit code 0" + check_all_benchmarks_ran "$label" + pids=$(distinct_pids) + [ "$pids" -eq 1 ] || fail "$label: expected 1 PID, got $pids" +done + +# 3. A failing benchmark fails the whole run, with or without isolation. +for setting in "true" "false"; do + label="failing, isolation=$setting" + FIXTURE_FAIL=1 CODSPEED_PROCESS_ISOLATION="$setting" run_fixture + code=$? + [ "$code" -eq "$FAILING_EXIT_CODE" ] || + fail "$label: expected exit code $FAILING_EXIT_CODE, got $code" +done + +# 4. The child ignores the command line filter. +CODSPEED_PROCESS_ISOLATION=true run_fixture --benchmark_filter=BM_First +[ $? -eq 0 ] || fail "filter: expected exit code 0" +grep -q "::BM_First" "$ERROR_FILE" || fail "filter: BM_First did not run" +grep -q "::BM_Second" "$ERROR_FILE" && fail "filter: BM_Second ran despite the filter" + +# 5. An invalid value is a hard error. +CODSPEED_PROCESS_ISOLATION=maybe run_fixture +[ $? -ne 0 ] || fail "invalid value: expected a non-zero exit code" +grep -q "CODSPEED_PROCESS_ISOLATION" "$ERROR_FILE" || fail "invalid value: no error message" + +if [ "$failures" -ne 0 ]; then + exit 1 +fi +echo "process isolation test passed"