diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af09a5b..80c1452 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,6 +102,22 @@ jobs: mode: ${{ matrix.codspeed-mode }} run: examples/google_benchmark_cmake/build/benchmark_example + process-isolation-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + submodules: "recursive" + + - name: Build and run the process isolation test + run: | + mkdir -p tests/process_isolation/build + cd tests/process_isolation/build + cmake -DCODSPEED_MODE=simulation -DCODSPEED_STRICT_WARNINGS=ON .. + make -j + ctest --output-on-failure + bazel-integration-tests: strategy: matrix: diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index f0d5eb2..83a2d37 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -85,6 +85,7 @@ endif() add_library( codspeed src/codspeed.cpp + src/isolation.cpp src/measurement.cpp src/walltime.cpp src/uri.cpp diff --git a/core/include/isolation.h b/core/include/isolation.h new file mode 100644 index 0000000..8924c56 --- /dev/null +++ b/core/include/isolation.h @@ -0,0 +1,60 @@ +#ifndef CODSPEED_ISOLATION_H +#define CODSPEED_ISOLATION_H + +#include +#include + +namespace codspeed { + +// Isolation is on unless this is `0` or `false`. +extern const char* const PROCESS_ISOLATION_ENV; +// Set on each child process: the URI of the only benchmark it runs. +extern const char* const ISOLATED_BENCHMARK_ENV; + +// How the harness runs each benchmark. Every benchmark runs in a fresh copy of +// the benchmark executable by default, so the allocator state and threads that +// other benchmarks or setup code leave behind can't change its measurement. +class Isolation { + public: + enum class Kind { + // All benchmarks share this process and its allocator state. + SharedProcess, + // Every benchmark runs in its own child process. + ProcessPerBenchmark, + // A child spawned by `ProcessPerBenchmark`, restricted to one URI. + IsolatedBenchmark, + }; + + // Stores the original command line. Call it before the arguments get + // parsed, because parsing removes the flags it consumed. The child process + // is started with the same arguments. + static void set_argv(int argc, const char* const* argv); + + // The isolation of this process, read once from the environment. + static const Isolation& current(); + + Kind kind() const { return kind_; } + + // The only benchmark this process may run, or null. It overrides CLI + // filters. + const std::string* only_benchmark() const; + + // Runs the benchmark `uri`. `bench` must contain the whole measurement, + // from `start_benchmark` to `end_benchmark`. + void run(const std::string& uri, const std::function& bench) const; + + // Runs this executable again with the same arguments, limited to `uri`, and + // exits with the child's code if it fails. + static void run_in_child_process(const std::string& uri); + + private: + Isolation(Kind kind, std::string isolated_uri); + static Isolation from_env(); + + Kind kind_; + std::string isolated_uri_; +}; + +} // namespace codspeed + +#endif // CODSPEED_ISOLATION_H diff --git a/core/src/isolation.cpp b/core/src/isolation.cpp new file mode 100644 index 0000000..fcacb7a --- /dev/null +++ b/core/src/isolation.cpp @@ -0,0 +1,212 @@ +#include "isolation.h" + +#include +#include +#include +#include +#include +#include + +#include "utils.h" + +#ifndef _WIN32 +#include +#include +#include + +#include +#ifdef __APPLE__ +#include +#endif + +extern char** environ; +#endif + +namespace codspeed { + +const char* const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const char* const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; + +namespace { + +// Exit code for a child that ended with neither an exit code nor a signal. +constexpr int GENERIC_FAILURE_EXIT_CODE = 1; +// Shells report a child killed by signal N as exit code 128 + N. +constexpr int SIGNAL_EXIT_CODE_OFFSET = 128; + +#ifndef _WIN32 +// Path of the running executable. `/proc/self/exe` keeps working when the +// benchmark changed its working directory or `argv[0]` is a bare name. +std::string current_executable() { +#if defined(__linux__) + return "/proc/self/exe"; +#elif defined(__APPLE__) + uint32_t size = 0; + _NSGetExecutablePath(nullptr, &size); + std::vector buffer(size); + if (_NSGetExecutablePath(buffer.data(), &size) == 0) { + return std::string(buffer.data()); + } + return std::string(); +#else + return std::string(); +#endif +} + +// The environment of this process plus the isolated benchmark URI. +std::vector child_environment(const std::string& uri) { + std::vector env; + const std::string isolated_prefix = std::string(ISOLATED_BENCHMARK_ENV) + "="; + for (char** entry = environ; entry != nullptr && *entry != nullptr; ++entry) { + if (std::strncmp(*entry, isolated_prefix.c_str(), isolated_prefix.size()) == + 0) { + continue; + } + env.emplace_back(*entry); + } + env.push_back(isolated_prefix + uri); + return env; +} + +// Exit code of a failed child, in the shell convention. +int exit_code(int wait_status) { + if (WIFSIGNALED(wait_status)) { + return SIGNAL_EXIT_CODE_OFFSET + WTERMSIG(wait_status); + } + if (WIFEXITED(wait_status)) { + return WEXITSTATUS(wait_status); + } + return GENERIC_FAILURE_EXIT_CODE; +} +#endif + +std::vector& original_argv() { + static std::vector argv; + return argv; +} + +} // namespace + +void Isolation::set_argv(int argc, const char* const* argv) { + auto& stored = original_argv(); + stored.clear(); + for (int i = 0; i < argc; ++i) { + stored.emplace_back(argv[i]); + } +} + +Isolation::Isolation(Kind kind, std::string isolated_uri) + : kind_(kind), isolated_uri_(std::move(isolated_uri)) {} + +const Isolation& Isolation::current() { + static const Isolation instance = from_env(); + return instance; +} + +Isolation Isolation::from_env() { + const std::string uri = safe_getenv(ISOLATED_BENCHMARK_ENV); + if (!uri.empty()) { + return Isolation(Kind::IsolatedBenchmark, uri); + } + + const std::string value = safe_getenv(PROCESS_ISOLATION_ENV); + if (value.empty() || value == "1" || value == "true") { + return Isolation(Kind::ProcessPerBenchmark, ""); + } + if (value == "0" || value == "false") { + return Isolation(Kind::SharedProcess, ""); + } + + std::cerr << PROCESS_ISOLATION_ENV << " must be `true` or `false`, got `" + << value << "`" << std::endl; + std::exit(GENERIC_FAILURE_EXIT_CODE); +} + +const std::string* Isolation::only_benchmark() const { + return kind_ == Kind::IsolatedBenchmark ? &isolated_uri_ : nullptr; +} + +void Isolation::run(const std::string& uri, + const std::function& bench) const { + switch (kind_) { + case Kind::SharedProcess: + case Kind::IsolatedBenchmark: + bench(); + return; + case Kind::ProcessPerBenchmark: + run_in_child_process(uri); + return; + } +} + +#ifdef _WIN32 +void Isolation::run_in_child_process(const std::string& uri) { + std::cerr << "process isolation is not supported on Windows, set " + << PROCESS_ISOLATION_ENV << "=false (benchmark: " << uri << ")" + << std::endl; + std::exit(GENERIC_FAILURE_EXIT_CODE); +} +#else +void Isolation::run_in_child_process(const std::string& uri) { + // Keep the parent's output before the child's. + std::fflush(stdout); + std::cout.flush(); + std::cerr.flush(); + + const std::string exe = current_executable(); + const auto& args = original_argv(); + if (exe.empty() && args.empty()) { + std::cerr << "failed to locate the benchmark executable for " << uri + << std::endl; + std::exit(GENERIC_FAILURE_EXIT_CODE); + } + + // Fall back to argv[0] where the OS gives no executable path. + const std::string& program = exe.empty() ? args[0] : exe; + + std::vector argv_storage = args; + if (argv_storage.empty()) { + argv_storage.push_back(program); + } + std::vector argv; + for (auto& arg : argv_storage) { + argv.push_back(&arg[0]); + } + argv.push_back(nullptr); + + std::vector env_storage = child_environment(uri); + std::vector envp; + for (auto& entry : env_storage) { + envp.push_back(&entry[0]); + } + envp.push_back(nullptr); + + // Re-exec instead of fork(): fork only copies the calling thread, so threads + // started before the harness (e.g. in main) would be missing in the child. + pid_t pid = 0; + const int spawn_error = posix_spawn(&pid, program.c_str(), nullptr, nullptr, + argv.data(), envp.data()); + if (spawn_error != 0) { + std::cerr << "failed to start the process for " << uri << ": " + << std::strerror(spawn_error) << std::endl; + std::exit(GENERIC_FAILURE_EXIT_CODE); + } + + int wait_status = 0; + while (waitpid(pid, &wait_status, 0) < 0) { + if (errno == EINTR) { + continue; + } + std::cerr << "failed to wait for the process of " << uri << ": " + << std::strerror(errno) << std::endl; + std::exit(GENERIC_FAILURE_EXIT_CODE); + } + + if (WIFEXITED(wait_status) && WEXITSTATUS(wait_status) == 0) { + return; + } + std::exit(exit_code(wait_status)); +} +#endif + +} // namespace codspeed diff --git a/google_benchmark/src/benchmark.cc b/google_benchmark/src/benchmark.cc index 610a62c..20fd4f2 100644 --- a/google_benchmark/src/benchmark.cc +++ b/google_benchmark/src/benchmark.cc @@ -20,6 +20,7 @@ #include "internal_macros.h" #include "measurement.hpp" +#include "isolation.h" #ifndef BENCHMARK_OS_WINDOWS #if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) @@ -464,9 +465,19 @@ void RunBenchmarks(const std::vector& benchmarks, // Note the file_reporter can be null. BM_CHECK(display_reporter != nullptr); +#ifdef CODSPEED_ANALYSIS + const codspeed::Isolation& isolation = codspeed::Isolation::current(); + // Process isolation children only run one benchmark. The parent already + // printed the mode and the context. + const bool is_isolated_child = isolation.only_benchmark() != nullptr; +#endif + #ifdef CODSPEED_ENABLED auto& Err = display_reporter->GetErrorStream(); - Err << "Codspeed mode: " << CODSPEED_MODE_DISPLAY << "\n"; +#ifdef CODSPEED_ANALYSIS + if (!is_isolated_child) +#endif + Err << "Codspeed mode: " << CODSPEED_MODE_DISPLAY << "\n"; #endif // CODSPEED_ENABLED bool might_have_aggregates = FLAGS_benchmark_repetitions > 1; @@ -493,8 +504,18 @@ void RunBenchmarks(const std::vector& benchmarks, std::map per_family_reports; - if (display_reporter->ReportContext(context) && - ((file_reporter == nullptr) || file_reporter->ReportContext(context))) { +#ifdef CODSPEED_ANALYSIS + const bool context_reported = + is_isolated_child || + (display_reporter->ReportContext(context) && + ((file_reporter == nullptr) || file_reporter->ReportContext(context))); +#else + const bool context_reported = + display_reporter->ReportContext(context) && + ((file_reporter == nullptr) || file_reporter->ReportContext(context)); +#endif + + if (context_reported) { FlushStreams(display_reporter); FlushStreams(file_reporter); @@ -560,10 +581,26 @@ void RunBenchmarks(const std::vector& benchmarks, #ifdef CODSPEED_WALLTIME std::vector codspeed_walltime_data; +#endif +#ifdef CODSPEED_ANALYSIS + // The child of a benchmark runs all of its repetitions, so the parent + // starts only one child per benchmark. + std::vector isolated(runners.size(), false); #endif for (size_t repetition_index : repetition_indices) { internal::BenchmarkRunner& runner = runners[repetition_index]; +#ifdef CODSPEED_ANALYSIS + if (isolation.kind() == codspeed::Isolation::Kind::ProcessPerBenchmark) { + if (!isolated[repetition_index]) { + isolated[repetition_index] = true; + isolation.run(runner.GetBenchmarkName(), + [&runner]() { runner.DoOneRepetition(); }); + } + continue; + } +#endif + #ifdef CODSPEED_WALLTIME auto* codspeed = codspeed::CodSpeed::getInstance(); if (codspeed != nullptr && runner.IsFirstRepetition()) { @@ -954,6 +991,13 @@ void PrintDefaultHelp() { } void Initialize(int* argc, char** argv, void (*HelperPrintf)()) { +#ifdef CODSPEED_ANALYSIS + // Keep the full command line: parsing below removes the benchmark flags, and + // process isolation starts the children with the original arguments. + codspeed::Isolation::set_argv(*argc, argv); + // Reject an invalid CODSPEED_PROCESS_ISOLATION before running anything. + codspeed::Isolation::current(); +#endif internal::HelperPrintf = HelperPrintf; internal::ParseCommandLineFlags(argc, argv); internal::LogLevel() = FLAGS_v; diff --git a/google_benchmark/src/benchmark_register.cc b/google_benchmark/src/benchmark_register.cc index 9904b9d..3f5c75c 100644 --- a/google_benchmark/src/benchmark_register.cc +++ b/google_benchmark/src/benchmark_register.cc @@ -13,6 +13,9 @@ // limitations under the License. #include "benchmark_register.h" +#ifdef CODSPEED_ANALYSIS +#include "isolation.h" +#endif #ifndef BENCHMARK_OS_WINDOWS #if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) @@ -172,9 +175,17 @@ bool BenchmarkFamilies::FindBenchmarks( num_threads); const auto full_name = instance.name().str(); - if (full_name.rfind(kDisabledPrefix, 0) != 0 && - ((re.Match(full_name) && !is_negative_filter) || - (!re.Match(full_name) && is_negative_filter))) { + bool matches_filter = (re.Match(full_name) && !is_negative_filter) || + (!re.Match(full_name) && is_negative_filter); +#ifdef CODSPEED_ANALYSIS + // A process isolation child runs exactly one benchmark, whatever the + // command line filter says. + if (const std::string* only = + codspeed::Isolation::current().only_benchmark()) { + matches_filter = full_name == *only; + } +#endif + if (full_name.rfind(kDisabledPrefix, 0) != 0 && matches_filter) { benchmarks->push_back(std::move(instance)); ++per_family_instance_index; diff --git a/google_benchmark/src/benchmark_runner.h b/google_benchmark/src/benchmark_runner.h index e89ba87..5dc4d53 100644 --- a/google_benchmark/src/benchmark_runner.h +++ b/google_benchmark/src/benchmark_runner.h @@ -124,7 +124,7 @@ class BenchmarkRunner { void RunWarmUp(); -#ifdef CODSPEED_WALLTIME +#if defined(CODSPEED_WALLTIME) || defined(CODSPEED_ANALYSIS) public: inline std::string GetBenchmarkName() { return b.name().str(); diff --git a/tests/process_isolation/BUILD.bazel b/tests/process_isolation/BUILD.bazel new file mode 100644 index 0000000..256319d --- /dev/null +++ b/tests/process_isolation/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "process_isolation_fixture", + srcs = ["main.cpp"], + copts = ["-Wall", "-Wextra", "-Werror"], + linkopts = ["-pthread"], + deps = ["@codspeed_google_benchmark_compat//:benchmark"], +) + +# Run with --@codspeed_core//:codspeed_mode=simulation (or another analysis +# mode), process isolation does not exist in the other modes. +sh_test( + name = "process_isolation_test", + srcs = ["run_test.sh"], + args = ["$(location :process_isolation_fixture)"], + data = [":process_isolation_fixture"], +) diff --git a/tests/process_isolation/CMakeLists.txt b/tests/process_isolation/CMakeLists.txt new file mode 100644 index 0000000..ed1d4fa --- /dev/null +++ b/tests/process_isolation/CMakeLists.txt @@ -0,0 +1,46 @@ +cmake_minimum_required(VERSION 3.12) +include(FetchContent) + +project(codspeed_process_isolation_test VERSION 0.0.0 LANGUAGES CXX) + +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + add_compile_options(-Wall -Wextra -Werror) +endif() + +set(CMAKE_BUILD_TYPE RelWithDebInfo) + +# Keep ctest limited to the isolation test. +set(BENCHMARK_ENABLE_TESTING + OFF + CACHE BOOL + "Enable testing of the benchmark library." + FORCE +) +set(BENCHMARK_ENABLE_GTEST_TESTS + OFF + CACHE BOOL + "Enable testing of the benchmark library." + FORCE +) + +FetchContent_Declare( + google_benchmark + SOURCE_DIR ${CMAKE_SOURCE_DIR}/../../google_benchmark +) +FetchContent_MakeAvailable(google_benchmark) + +find_package(Threads REQUIRED) +add_executable(process_isolation_fixture main.cpp) +target_link_libraries( + process_isolation_fixture + PRIVATE benchmark::benchmark Threads::Threads +) + +# Process isolation only exists in the analysis modes. +enable_testing() +add_test( + NAME process_isolation + COMMAND + ${CMAKE_CURRENT_SOURCE_DIR}/run_test.sh + $ +) diff --git a/tests/process_isolation/main.cpp b/tests/process_isolation/main.cpp new file mode 100644 index 0000000..0ea33f1 --- /dev/null +++ b/tests/process_isolation/main.cpp @@ -0,0 +1,124 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Set to make `BM_Failing` exit with `FAILING_EXIT_CODE`. +static const char* const FAIL_ENV = "FIXTURE_FAIL"; +static constexpr int FAILING_EXIT_CODE = 3; + +// Started in `main`, before any benchmark, so its thread belongs to the +// process that runs the harness. A forked child would not have it. +class MainThreadWorker { + public: + MainThreadWorker() : thread_([this] { loop(); }) {} + + ~MainThreadWorker() { + { + std::lock_guard lock(mutex_); + done_ = true; + } + wakeup_.notify_one(); + thread_.join(); + } + + int submit(int value) { + std::promise result; + { + std::lock_guard lock(mutex_); + jobs_.push([&result, value] { result.set_value(value + 1); }); + } + wakeup_.notify_one(); + return result.get_future().get(); + } + + private: + void loop() { + std::unique_lock lock(mutex_); + while (true) { + wakeup_.wait(lock, [this] { return done_ || !jobs_.empty(); }); + if (jobs_.empty()) { + return; + } + auto job = std::move(jobs_.front()); + jobs_.pop(); + lock.unlock(); + job(); + lock.lock(); + } + } + + std::mutex mutex_; + std::condition_variable wakeup_; + std::queue> jobs_; + bool done_ = false; + std::thread thread_; +}; + +static MainThreadWorker* worker = nullptr; + +// The test reads these lines to count the processes. +static void print_pid_once(const char* name) { + static std::mutex print_mutex; + std::lock_guard lock(print_mutex); + static std::string printed; + if (printed.find(name) != std::string::npos) { + return; + } + printed += name; + std::printf("PID %s %d\n", name, static_cast(getpid())); + std::fflush(stdout); +} + +static void BM_First(benchmark::State& state) { + print_pid_once("first"); + for (auto _ : state) { + benchmark::DoNotOptimize(1); + } +} +BENCHMARK(BM_First); + +static void BM_Second(benchmark::State& state) { + print_pid_once("second"); + for (auto _ : state) { + benchmark::DoNotOptimize(2); + } +} +BENCHMARK(BM_Second); + +static void BM_OnMainWorker(benchmark::State& state) { + print_pid_once("worker"); + for (auto _ : state) { + benchmark::DoNotOptimize(worker->submit(3)); + } +} +BENCHMARK(BM_OnMainWorker); + +static void BM_Failing(benchmark::State& state) { + print_pid_once("failing"); + for (auto _ : state) { + std::exit(FAILING_EXIT_CODE); + } +} + +int main(int argc, char** argv) { + MainThreadWorker main_worker; + worker = &main_worker; + + if (std::getenv(FAIL_ENV) != nullptr) { + benchmark::RegisterBenchmark("BM_Failing", BM_Failing); + } + + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + return 0; +} diff --git a/tests/process_isolation/run_test.sh b/tests/process_isolation/run_test.sh new file mode 100755 index 0000000..f739b24 --- /dev/null +++ b/tests/process_isolation/run_test.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# End to end test of process isolation: runs the fixture binary, built in an +# analysis mode, and checks the processes it used and its exit code. +set -u + +FIXTURE="$1" +EXPECTED_BENCHMARKS=(BM_First BM_Second BM_OnMainWorker) +FAILING_EXIT_CODE=3 +TIMEOUT_SECONDS=120 + +failures=0 +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} + +run_fixture() { + timeout "$TIMEOUT_SECONDS" "$FIXTURE" "$@" >"$OUTPUT_FILE" 2>"$ERROR_FILE" + return $? +} + +OUTPUT_FILE="$(mktemp)" +ERROR_FILE="$(mktemp)" +trap 'rm -f "$OUTPUT_FILE" "$ERROR_FILE"' EXIT + +distinct_pids() { + grep '^PID ' "$OUTPUT_FILE" | awk '{print $3}' | sort -u | wc -l +} + +check_all_benchmarks_ran() { + local label="$1" + for name in "${EXPECTED_BENCHMARKS[@]}"; do + grep -q "::${name}" "$ERROR_FILE" || fail "$label: no 'Measured/Checked' line for $name" + done +} + +# 1. Isolation on (default and explicit): one process per benchmark. +for setting in "" "true" "1"; do + label="isolation=${setting:-}" + if [ -z "$setting" ]; then + env -u CODSPEED_PROCESS_ISOLATION -u FIXTURE_FAIL \ + timeout "$TIMEOUT_SECONDS" "$FIXTURE" >"$OUTPUT_FILE" 2>"$ERROR_FILE" + else + CODSPEED_PROCESS_ISOLATION="$setting" run_fixture + fi + [ $? -eq 0 ] || fail "$label: expected exit code 0" + check_all_benchmarks_ran "$label" + pids=$(distinct_pids) + [ "$pids" -eq "${#EXPECTED_BENCHMARKS[@]}" ] || + fail "$label: expected ${#EXPECTED_BENCHMARKS[@]} distinct PIDs, got $pids" + # The parent prints the context once, children must stay quiet. + [ "$(grep -c 'Run on' "$ERROR_FILE")" -le 1 ] || fail "$label: context printed more than once" +done + +# 2. Isolation off: all benchmarks share one process. +for setting in "false" "0"; do + label="isolation=$setting" + CODSPEED_PROCESS_ISOLATION="$setting" run_fixture + [ $? -eq 0 ] || fail "$label: expected exit code 0" + check_all_benchmarks_ran "$label" + pids=$(distinct_pids) + [ "$pids" -eq 1 ] || fail "$label: expected 1 PID, got $pids" +done + +# 3. A failing benchmark fails the whole run, with or without isolation. +for setting in "true" "false"; do + label="failing, isolation=$setting" + FIXTURE_FAIL=1 CODSPEED_PROCESS_ISOLATION="$setting" run_fixture + code=$? + [ "$code" -eq "$FAILING_EXIT_CODE" ] || + fail "$label: expected exit code $FAILING_EXIT_CODE, got $code" +done + +# 4. The child ignores the command line filter. +CODSPEED_PROCESS_ISOLATION=true run_fixture --benchmark_filter=BM_First +[ $? -eq 0 ] || fail "filter: expected exit code 0" +grep -q "::BM_First" "$ERROR_FILE" || fail "filter: BM_First did not run" +grep -q "::BM_Second" "$ERROR_FILE" && fail "filter: BM_Second ran despite the filter" + +# 5. An invalid value is a hard error. +CODSPEED_PROCESS_ISOLATION=maybe run_fixture +[ $? -ne 0 ] || fail "invalid value: expected a non-zero exit code" +grep -q "CODSPEED_PROCESS_ISOLATION" "$ERROR_FILE" || fail "invalid value: no error message" + +if [ "$failures" -ne 0 ]; then + exit 1 +fi +echo "process isolation test passed"