feat: run each benchmark in its own process

.github/workflows/ci.yml

@@ -102,6 +102,22 @@ jobs:
           mode: ${{ matrix.codspeed-mode }}
           run: examples/google_benchmark_cmake/build/benchmark_example
 
+  process-isolation-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Build and run the process isolation test
+        run: |
+          mkdir -p tests/process_isolation/build
+          cd tests/process_isolation/build
+          cmake -DCODSPEED_MODE=simulation -DCODSPEED_STRICT_WARNINGS=ON ..
+          make -j
+          ctest --output-on-failure
+
   bazel-integration-tests:
     strategy:
       matrix:

core/CMakeLists.txt

@@ -85,6 +85,7 @@ endif()
 add_library(
     codspeed
     src/codspeed.cpp
+    src/isolation.cpp
     src/measurement.cpp
     src/walltime.cpp
     src/uri.cpp

core/include/isolation.h

@@ -0,0 +1,60 @@
+#ifndef CODSPEED_ISOLATION_H
+#define CODSPEED_ISOLATION_H
+
+#include <functional>
+#include <string>
+
+namespace codspeed {
+
+// Isolation is on unless this is `0` or `false`.
+extern const char* const PROCESS_ISOLATION_ENV;
+// Set on each child process: the URI of the only benchmark it runs.
+extern const char* const ISOLATED_BENCHMARK_ENV;
+
+// How the harness runs each benchmark. Every benchmark runs in a fresh copy of
+// the benchmark executable by default, so the allocator state and threads that
+// other benchmarks or setup code leave behind can't change its measurement.
+class Isolation {
+ public:
+  enum class Kind {
+    // All benchmarks share this process and its allocator state.
+    SharedProcess,
+    // Every benchmark runs in its own child process.
+    ProcessPerBenchmark,
+    // A child spawned by `ProcessPerBenchmark`, restricted to one URI.
+    IsolatedBenchmark,
+  };
+
+  // Stores the original command line. Call it before the arguments get
+  // parsed, because parsing removes the flags it consumed. The child process
+  // is started with the same arguments.
+  static void set_argv(int argc, const char* const* argv);
+
+  // The isolation of this process, read once from the environment.
+  static const Isolation& current();
+
+  Kind kind() const { return kind_; }
+
+  // The only benchmark this process may run, or null. It overrides CLI
+  // filters.
+  const std::string* only_benchmark() const;
+
+  // Runs the benchmark `uri`. `bench` must contain the whole measurement,
+  // from `start_benchmark` to `end_benchmark`.
+  void run(const std::string& uri, const std::function<void()>& bench) const;
+
+  // Runs this executable again with the same arguments, limited to `uri`, and
+  // exits with the child's code if it fails.
+  static void run_in_child_process(const std::string& uri);
+
+ private:
+  Isolation(Kind kind, std::string isolated_uri);
+  static Isolation from_env();
+
+  Kind kind_;
+  std::string isolated_uri_;
+};
+
+}  // namespace codspeed
+
+#endif  // CODSPEED_ISOLATION_H

core/src/isolation.cpp

@@ -0,0 +1,212 @@
+#include "isolation.h"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "utils.h"
+
+#ifndef _WIN32
+#include <spawn.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <cerrno>
+#ifdef __APPLE__
+#include <mach-o/dyld.h>
+#endif
+
+extern char** environ;
+#endif
+
+namespace codspeed {
+
+const char* const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION";
+const char* const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK";
+
+namespace {
+
+// Exit code for a child that ended with neither an exit code nor a signal.
+constexpr int GENERIC_FAILURE_EXIT_CODE = 1;
+// Shells report a child killed by signal N as exit code 128 + N.
+constexpr int SIGNAL_EXIT_CODE_OFFSET = 128;
+
+#ifndef _WIN32
+// Path of the running executable. `/proc/self/exe` keeps working when the
+// benchmark changed its working directory or `argv[0]` is a bare name.
+std::string current_executable() {
+#if defined(__linux__)
+  return "/proc/self/exe";
+#elif defined(__APPLE__)
+  uint32_t size = 0;
+  _NSGetExecutablePath(nullptr, &size);
+  std::vector<char> buffer(size);
+  if (_NSGetExecutablePath(buffer.data(), &size) == 0) {
+    return std::string(buffer.data());
+  }
+  return std::string();
+#else
+  return std::string();
+#endif
+}
+
+// The environment of this process plus the isolated benchmark URI.
+std::vector<std::string> child_environment(const std::string& uri) {
+  std::vector<std::string> env;
+  const std::string isolated_prefix = std::string(ISOLATED_BENCHMARK_ENV) + "=";
+  for (char** entry = environ; entry != nullptr && *entry != nullptr; ++entry) {
+    if (std::strncmp(*entry, isolated_prefix.c_str(), isolated_prefix.size()) ==
+        0) {
+      continue;
+    }
+    env.emplace_back(*entry);
+  }
+  env.push_back(isolated_prefix + uri);
+  return env;
+}
+
+// Exit code of a failed child, in the shell convention.
+int exit_code(int wait_status) {
+  if (WIFSIGNALED(wait_status)) {
+    return SIGNAL_EXIT_CODE_OFFSET + WTERMSIG(wait_status);
+  }
+  if (WIFEXITED(wait_status)) {
+    return WEXITSTATUS(wait_status);
+  }
+  return GENERIC_FAILURE_EXIT_CODE;
+}
+#endif
+
+std::vector<std::string>& original_argv() {
+  static std::vector<std::string> argv;
+  return argv;
+}
+
+}  // namespace
+
+void Isolation::set_argv(int argc, const char* const* argv) {
+  auto& stored = original_argv();
+  stored.clear();
+  for (int i = 0; i < argc; ++i) {
+    stored.emplace_back(argv[i]);
+  }
+}
+
+Isolation::Isolation(Kind kind, std::string isolated_uri)
+    : kind_(kind), isolated_uri_(std::move(isolated_uri)) {}
+
+const Isolation& Isolation::current() {
+  static const Isolation instance = from_env();
+  return instance;
+}
+
+Isolation Isolation::from_env() {
+  const std::string uri = safe_getenv(ISOLATED_BENCHMARK_ENV);
+  if (!uri.empty()) {
+    return Isolation(Kind::IsolatedBenchmark, uri);
+  }
+
+  const std::string value = safe_getenv(PROCESS_ISOLATION_ENV);
+  if (value.empty() || value == "1" || value == "true") {
+    return Isolation(Kind::ProcessPerBenchmark, "");
+  }
+  if (value == "0" || value == "false") {
+    return Isolation(Kind::SharedProcess, "");
+  }
+
+  std::cerr << PROCESS_ISOLATION_ENV << " must be `true` or `false`, got `"
+            << value << "`" << std::endl;
+  std::exit(GENERIC_FAILURE_EXIT_CODE);
+}
+
+const std::string* Isolation::only_benchmark() const {
+  return kind_ == Kind::IsolatedBenchmark ? &isolated_uri_ : nullptr;
+}
+
+void Isolation::run(const std::string& uri,
+                    const std::function<void()>& bench) const {
+  switch (kind_) {
+    case Kind::SharedProcess:
+    case Kind::IsolatedBenchmark:
+      bench();
+      return;
+    case Kind::ProcessPerBenchmark:
+      run_in_child_process(uri);
+      return;
+  }
+}
+
+#ifdef _WIN32
+void Isolation::run_in_child_process(const std::string& uri) {
+  std::cerr << "process isolation is not supported on Windows, set "
+            << PROCESS_ISOLATION_ENV << "=false (benchmark: " << uri << ")"
+            << std::endl;
+  std::exit(GENERIC_FAILURE_EXIT_CODE);
+}
+#else
+void Isolation::run_in_child_process(const std::string& uri) {
+  // Keep the parent's output before the child's.
+  std::fflush(stdout);
+  std::cout.flush();
+  std::cerr.flush();
+
+  const std::string exe = current_executable();
+  const auto& args = original_argv();
+  if (exe.empty() && args.empty()) {
+    std::cerr << "failed to locate the benchmark executable for " << uri
+              << std::endl;
+    std::exit(GENERIC_FAILURE_EXIT_CODE);
+  }
+
+  // Fall back to argv[0] where the OS gives no executable path.
+  const std::string& program = exe.empty() ? args[0] : exe;
+
+  std::vector<std::string> argv_storage = args;
+  if (argv_storage.empty()) {
+    argv_storage.push_back(program);
+  }
+  std::vector<char*> argv;
+  for (auto& arg : argv_storage) {
+    argv.push_back(&arg[0]);
+  }
+  argv.push_back(nullptr);
+
+  std::vector<std::string> env_storage = child_environment(uri);
+  std::vector<char*> envp;
+  for (auto& entry : env_storage) {
+    envp.push_back(&entry[0]);
+  }
+  envp.push_back(nullptr);
+
+  // Re-exec instead of fork(): fork only copies the calling thread, so threads
+  // started before the harness (e.g. in main) would be missing in the child.
+  pid_t pid = 0;
+  const int spawn_error = posix_spawn(&pid, program.c_str(), nullptr, nullptr,
+                                      argv.data(), envp.data());
+  if (spawn_error != 0) {
+    std::cerr << "failed to start the process for " << uri << ": "
+              << std::strerror(spawn_error) << std::endl;
+    std::exit(GENERIC_FAILURE_EXIT_CODE);
+  }
+
+  int wait_status = 0;
+  while (waitpid(pid, &wait_status, 0) < 0) {
+    if (errno == EINTR) {
+      continue;
+    }
+    std::cerr << "failed to wait for the process of " << uri << ": "
+              << std::strerror(errno) << std::endl;
+    std::exit(GENERIC_FAILURE_EXIT_CODE);
+  }
+
+  if (WIFEXITED(wait_status) && WEXITSTATUS(wait_status) == 0) {
+    return;
+  }
+  std::exit(exit_code(wait_status));
+}
+#endif
+
+}  // namespace codspeed

google_benchmark/src/benchmark.cc

@@ -20,6 +20,7 @@
 #include "internal_macros.h"
 
 #include "measurement.hpp"
+#include "isolation.h"
 
 #ifndef BENCHMARK_OS_WINDOWS
 #if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT)
@@ -464,9 +465,19 @@ void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
   // Note the file_reporter can be null.
   BM_CHECK(display_reporter != nullptr);
 
+#ifdef CODSPEED_ANALYSIS
+  const codspeed::Isolation& isolation = codspeed::Isolation::current();
+  // Process isolation children only run one benchmark. The parent already
+  // printed the mode and the context.
+  const bool is_isolated_child = isolation.only_benchmark() != nullptr;
+#endif
+
 #ifdef CODSPEED_ENABLED
   auto& Err = display_reporter->GetErrorStream();
-  Err << "Codspeed mode: " << CODSPEED_MODE_DISPLAY << "\n";
+#ifdef CODSPEED_ANALYSIS
+  if (!is_isolated_child)
+#endif
+    Err << "Codspeed mode: " << CODSPEED_MODE_DISPLAY << "\n";
 #endif  // CODSPEED_ENABLED
 
   bool might_have_aggregates = FLAGS_benchmark_repetitions > 1;
@@ -493,8 +504,18 @@ void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
   std::map<int /*family_index*/, BenchmarkReporter::PerFamilyRunReports>
       per_family_reports;
 
-  if (display_reporter->ReportContext(context) &&
-      ((file_reporter == nullptr) || file_reporter->ReportContext(context))) {
+#ifdef CODSPEED_ANALYSIS
+  const bool context_reported =
+      is_isolated_child ||
+      (display_reporter->ReportContext(context) &&
+       ((file_reporter == nullptr) || file_reporter->ReportContext(context)));
+#else
+  const bool context_reported =
+      display_reporter->ReportContext(context) &&
+      ((file_reporter == nullptr) || file_reporter->ReportContext(context));
+#endif
+
+  if (context_reported) {
     FlushStreams(display_reporter);
     FlushStreams(file_reporter);
 
@@ -560,10 +581,26 @@ void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
 
 #ifdef CODSPEED_WALLTIME
     std::vector<codspeed::RawWalltimeBenchmark> codspeed_walltime_data;
+#endif
+#ifdef CODSPEED_ANALYSIS
+    // The child of a benchmark runs all of its repetitions, so the parent
+    // starts only one child per benchmark.
+    std::vector<bool> isolated(runners.size(), false);
 #endif
     for (size_t repetition_index : repetition_indices) {
       internal::BenchmarkRunner& runner = runners[repetition_index];
 
+#ifdef CODSPEED_ANALYSIS
+      if (isolation.kind() == codspeed::Isolation::Kind::ProcessPerBenchmark) {
+        if (!isolated[repetition_index]) {
+          isolated[repetition_index] = true;
+          isolation.run(runner.GetBenchmarkName(),
+                        [&runner]() { runner.DoOneRepetition(); });
+        }
+        continue;
+      }
+#endif
+
 #ifdef CODSPEED_WALLTIME
       auto* codspeed = codspeed::CodSpeed::getInstance();
       if (codspeed != nullptr && runner.IsFirstRepetition()) {
@@ -954,6 +991,13 @@ void PrintDefaultHelp() {
 }
 
 void Initialize(int* argc, char** argv, void (*HelperPrintf)()) {
+#ifdef CODSPEED_ANALYSIS
+  // Keep the full command line: parsing below removes the benchmark flags, and
+  // process isolation starts the children with the original arguments.
+  codspeed::Isolation::set_argv(*argc, argv);
+  // Reject an invalid CODSPEED_PROCESS_ISOLATION before running anything.
+  codspeed::Isolation::current();
+#endif
   internal::HelperPrintf = HelperPrintf;
   internal::ParseCommandLineFlags(argc, argv);
   internal::LogLevel() = FLAGS_v;