From f5180724ba53970c2ad2d3ad973fe3f63ed558f5 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:20:26 +0900 Subject: [PATCH 01/19] fix(windows): don't require symlinks for the runtime venv If creating a symlink fails, e.g. on Windows without the privilege to create symlinks, the venv that is created at runtime uses hard links, junctions or copies instead, and its pyvenv.cfg sets home, because Python can't find it from the interpreter otherwise. The RULES_PYTHON_TESTING_DISABLE_SYMLINKS environment variable forces the fallback, so that it can be tested where symlinks work. --- news/3792.fixed.md | 4 + python/private/python_bootstrap_template.txt | 77 +++++++++++++++++-- tests/bootstrap_impls/BUILD.bazel | 12 ++- .../run_binary_without_symlinks_test.sh | 59 ++++++++++++++ 4 files changed, 143 insertions(+), 9 deletions(-) create mode 100644 news/3792.fixed.md create mode 100755 tests/bootstrap_impls/run_binary_without_symlinks_test.sh diff --git a/news/3792.fixed.md b/news/3792.fixed.md new file mode 100644 index 0000000000..01cdc76da4 --- /dev/null +++ b/news/3792.fixed.md @@ -0,0 +1,4 @@ +(binaries/tests) (Windows) `py_binary` and `py_test` no longer need the +privilege to create symlinks to run. If creating a symlink fails, the venv +that is created at runtime uses hard links, junctions or copies instead +([#3792](https://github.com/bazel-contrib/rules_python/issues/3792)). diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 8d95f9d419..7d6d0a6656 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -453,7 +453,11 @@ print(site.getsitepackages(["{venv_src}"])[-1]) if os.path.exists(runfiles_venv_lib): _symlink_tree(link_from=venv_lib, link_to=runfiles_venv_lib) - if IS_WINDOWS: + if IS_WINDOWS or not _SYMLINKS_SUPPORTED: + if python_home is None: + # Without a symlink, Python can't find home from the venv + # interpreter, so it has to be in pyvenv.cfg. + python_home = dirname(python_exe_actual) print_verbose("create_venv: pyvenv.cfg home: ", python_home) venv_pyvenv_cfg = join(venv, "pyvenv.cfg") with open(venv_pyvenv_cfg, "w") as fp: @@ -599,7 +603,7 @@ def execute_file(python_program, main_filename, args, env, runfiles_root, if delete_dirs: for delete_dir in delete_dirs: print_verbose("cleanup: rmtree:", delete_dir) - shutil.rmtree(delete_dir, True) + _rmtree(delete_dir) sys.exit(ret_code) def _run_execv(python_program, argv, env): @@ -616,14 +620,71 @@ def _run_execv(python_program, argv, env): print_verbose("pyprog head:" + str(f.read(50))) raise +# Set to False once creating a symlink fails, e.g. on Windows without the +# privilege to create symlinks, so that it isn't attempted for every file. +# Tests set the environment variable to use the fallback even when symlinks +# can be created. +_SYMLINKS_SUPPORTED = not os.environ.get("RULES_PYTHON_TESTING_DISABLE_SYMLINKS") + def _symlink_exist_ok(*, from_, to): + """Creates a symlink, or a hard link, junction or copy if that's not possible.""" + global _SYMLINKS_SUPPORTED + # On Windows, symlinks have to be told whether they're + # pointing to a file or directory. Python is supposed to auto-detect + # this, but in practice, this doesn't reliably happen. + is_dir = os.path.isdir(to) + if _SYMLINKS_SUPPORTED: + try: + os.symlink(to, from_, target_is_directory=is_dir) + return + except FileExistsError: + return + except OSError as e: + print_verbose("symlink failed, falling back to links or copies:", e) + _SYMLINKS_SUPPORTED = False + + if os.path.lexists(from_): + return + if is_dir: + _link_dir(from_, to) + else: + _link_file(from_, to) + +def _link_file(from_, to): + # Hard links don't need any privileges, but only work within the same + # volume, so fall back to copying the file. try: - # On Windows, symlinks have to be told whether they're - # pointing to a file or directory. Python is supposed to auto-detect - # this, but in practice, this doesn't reliably happen. - os.symlink(to, from_, target_is_directory=os.path.isdir(to)) - except FileExistsError: - pass + os.link(to, from_) + except OSError as e: + print_verbose("hard link failed, copying:", to, "error:", e) + shutil.copy2(to, from_) + +def _link_dir(from_, to): + if IS_WINDOWS: + try: + # Junctions don't need any privileges, but need an absolute target. + import _winapi + _winapi.CreateJunction(abspath(to), from_) + return + except (ImportError, AttributeError, OSError) as e: + print_verbose("junction failed, copying:", to, "error:", e) + shutil.copytree(to, from_) + +def _rmtree(path): + """Removes a directory, including read-only files, e.g. copies of outputs.""" + def _on_error(func, error_path, _): + # Files copied or hard linked from Bazel outputs are read-only and + # Windows refuses to delete them. + try: + os.chmod(error_path, 0o777) + func(error_path) + except OSError: + pass + + if sys.version_info >= (3, 12): + shutil.rmtree(path, onexc=_on_error) + else: + shutil.rmtree(path, onerror=_on_error) def main(): diff --git a/tests/bootstrap_impls/BUILD.bazel b/tests/bootstrap_impls/BUILD.bazel index f5c8c1fe82..d95dbd0be1 100644 --- a/tests/bootstrap_impls/BUILD.bazel +++ b/tests/bootstrap_impls/BUILD.bazel @@ -17,7 +17,7 @@ load("//python:py_test.bzl", "py_test") load("//python:pytest_test.bzl", "pytest_test") load("//tests/support:py_reconfig.bzl", "py_reconfig_binary", "py_reconfig_test") load("//tests/support:sh_py_run_test.bzl", "sh_py_run_test") -load("//tests/support:support.bzl", "SUPPORTS_BOOTSTRAP_SCRIPT") +load("//tests/support:support.bzl", "BAZEL_8_OR_LATER", "SUPPORTS_BOOTSTRAP_SCRIPT") load(":venv_relative_path_tests.bzl", "relative_path_test_suite") py_reconfig_binary( @@ -73,6 +73,16 @@ sh_py_run_test( venvs_use_declare_symlink = "no", ) +sh_py_run_test( + name = "run_binary_without_symlinks_test", + bootstrap_impl = "system_python", + py_src = "bin.py", + sh_src = "run_binary_without_symlinks_test.sh", + # Bazel 7 doesn't create a full venv for bootstrap_impl=system_python. + target_compatible_with = BAZEL_8_OR_LATER, + venvs_use_declare_symlink = "no", +) + sh_py_run_test( name = "run_binary_find_runfiles_test", py_src = "bin.py", diff --git a/tests/bootstrap_impls/run_binary_without_symlinks_test.sh b/tests/bootstrap_impls/run_binary_without_symlinks_test.sh new file mode 100755 index 0000000000..b056eda29b --- /dev/null +++ b/tests/bootstrap_impls/run_binary_without_symlinks_test.sh @@ -0,0 +1,59 @@ +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- +set +e + +bin=$(rlocation $BIN_RLOCATION) +if [[ -z "$bin" ]]; then + echo "Unable to locate test binary: $BIN_RLOCATION" + exit 1 +fi + +function expect_match() { + local expected_pattern=$1 + local actual=$2 + if ! (echo "$actual" | grep "$expected_pattern" ) >/dev/null; then + echo "expected to match: $expected_pattern" + echo "===== actual START =====" + echo "$actual" + echo "===== actual END =====" + echo + touch EXPECTATION_FAILED + return 1 + fi +} + +# Use hard links, junctions or copies instead of symlinks for the venv that +# is created at runtime, like on Windows without the privilege to create +# symlinks. +venvs_root=$(mktemp -d) +actual=$(RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1 RULES_PYTHON_EXTRACT_ROOT=$venvs_root $bin) +expect_match "Hello" "$actual" +expect_match "sys.executable:.*$venvs_root" "$actual" + +venv_python=$(echo "$actual" | grep "^sys.executable:" | cut -d' ' -f2-) +if [[ -L "$venv_python" ]]; then + echo "expected the venv interpreter not to be a symlink: $venv_python" + touch EXPECTATION_FAILED +fi + +# Without RULES_PYTHON_EXTRACT_ROOT, the venv is removed after the run, which +# must work for read-only copies of Bazel outputs too. +actual=$(RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1 $bin) +expect_match "Hello" "$actual" +venv_python=$(echo "$actual" | grep "^sys.executable:" | cut -d' ' -f2-) +venv=$(dirname "$(dirname "$venv_python")") +if [[ -e "$venv" ]]; then + echo "expected the venv to be removed: $venv" + touch EXPECTATION_FAILED +fi + +# Exit if any of the expects failed +[[ ! -e EXPECTATION_FAILED ]] From 056ae8cfcac90abc6f08b06ba15b44916637a85d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:20:30 +0900 Subject: [PATCH 02/19] ci: test examples/bzlmod on Windows without symlinks --- .bazelci/presubmit.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 2f7bdd96ed..d7be592422 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -394,6 +394,25 @@ tasks: - "bazel build --vendor_dir=./vendor //..." - "rm -rf ./vendor" + integration_test_bzlmod_windows_no_symlinks: + <<: *reusable_build_test_all + name: "examples/bzlmod: Windows, without symlinks" + working_directory: examples/bzlmod + platform: windows + bazel: 8.x + batch_commands: + # The last value wins, so this overrides the setting in .bazelrc to + # check that binaries work without the privilege to create symlinks. + - "echo startup --nowindows_enable_symlinks >> .bazelrc" + build_flags: + - "--keep_going" + # The CI user may be able to create symlinks, so force the fallback. + - "--action_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" + test_flags: + - "--keep_going" + - "--action_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" + - "--test_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" + integration_test_multi_python_versions_ubuntu_workspace: <<: *reusable_build_test_all <<: *common_workspace_flags From 0a739bf4e1d8fc177b34383f31e6121f87ee851e Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:57:13 +0900 Subject: [PATCH 03/19] ci: temporarily run only the Windows jobs Revert this before the PR leaves draft. --- .bazelci/presubmit.yml | 382 +---------------------------------------- 1 file changed, 1 insertion(+), 381 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index d7be592422..52680fee0e 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -151,7 +151,7 @@ tasks: # LINT.IfChange(bcr_test) bcr_test: name: "BCR: Bazel {bazel}" - platform: ${{ platform }} + platform: windows working_directory: examples/bzlmod bazel: ${{ bazel }} build_flags: @@ -165,116 +165,8 @@ tasks: # LINT.ThenChange(//.bcr/presubmit.yml:tasks) # LINT.IfChange(gazelle_bcr) - gazelle_bcr_ubuntu: - <<: *gazelle_common_bcr - name: "Gazelle: BCR, Bazel {bazel}" - platform: ubuntu2204 - gazelle_bcr_debian: - <<: *gazelle_common_bcr - name: "Gazelle: BCR, Bazel {bazel}" - platform: debian13 - gazelle_bcr_macos_arm64: - <<: *gazelle_common_bcr - name: "Gazelle: BCR, Bazel {bazel}" - platform: macos_arm64 # LINT.ThenChange(//.bcr/gazelle/presubmit.yml:matrix) - gazelle_extension_min: - <<: *common_workspace_flags_min_bazel - <<: *minimum_supported_version - name: "Gazelle: workspace, minimum supported Bazel version" - platform: ubuntu2204 - build_targets: ["//..."] - test_targets: ["//..."] - working_directory: gazelle - gazelle_extension_workspace: - <<: *common_workspace_flags - name: "Gazelle: workspace" - platform: ubuntu2204 - build_targets: ["//..."] - test_targets: ["//..."] - working_directory: gazelle - gazelle_extension: - name: "Gazelle: default settings" - platform: ubuntu2204 - build_targets: ["//..."] - test_targets: ["//..."] - working_directory: gazelle - - sphinxdocs_ubuntu: - name: "Sphinxdocs: Ubuntu" - working_directory: "sphinxdocs" - platform: ubuntu2204 - shell_commands: - - "../.bazelci/isolate.sh sphinxdocs" - build_targets: - - "//..." - test_targets: - - "//..." - sphinxdocs_mac: - name: "Sphinxdocs: Mac" - working_directory: "sphinxdocs" - platform: macos_arm64 - shell_commands: - - "../.bazelci/isolate.sh sphinxdocs" - build_targets: - - "//..." - test_targets: - - "//..." - - ubuntu_min_workspace: - <<: *minimum_supported_version - <<: *reusable_config - <<: *common_workspace_flags_min_bazel - name: "Default: Ubuntu, workspace, minimum Bazel" - platform: ubuntu2204 - - ubuntu_min_bzlmod: - <<: *minimum_supported_version - <<: *reusable_config - name: "Default: Ubuntu, bzlmod, minimum Bazel" - platform: ubuntu2204 - bazel: 7.x - ubuntu: - <<: *reusable_config - <<: *coverage_targets_bootstrap - name: "Default: Ubuntu, Bazel {bazel}" - platform: ubuntu2204 - bazel: ${{ bazel }} - ubuntu_upcoming: - <<: *reusable_config - name: "Default: Ubuntu, upcoming Bazel" - platform: ubuntu2204 - bazel: last_rc - # This is an advisory job; doesn't block merges - # RCs may have regressions, so don't fail our CI on them - soft_fail: - - exit_status: 1 - - exit_status: 3 - ubuntu_rolling: - name: "Default: Ubuntu, rolling Bazel" - platform: ubuntu2204 - bazel: rolling - # This is an advisory job; doesn't block merges - soft_fail: - - exit_status: 1 - - exit_status: 3 - test_targets: - - "--" - - "//tests/..." - test_flags: - <<: *reusable_config_test_flags - ? "--keep_going" - ubuntu_workspace: - <<: *reusable_config - <<: *common_workspace_flags - name: "Default: Ubuntu, workspace" - platform: ubuntu2204 - mac_workspace: - <<: *reusable_config - <<: *common_workspace_flags - name: "Default: Mac, workspace" - platform: macos_arm64 windows_workspace: <<: *reusable_config <<: *common_workspace_flags @@ -300,17 +192,6 @@ tasks: ? "--noenable_bzlmod" ? "--enable_workspace" ? "--test_tag_filters=-integration-test" - - debian: - <<: *reusable_config - name: "Default: Debian, Bazel {bazel}" - platform: debian13 - bazel: ${{ bazel }} - macos_arm64: - <<: *reusable_config - name: "Default: MacOS, Bazel {bazel}" - platform: macos_arm64 - bazel: ${{ bazel }} windows: <<: *reusable_config name: "Default: Windows, Bazel {bazel}" @@ -319,64 +200,6 @@ tasks: test_flags: <<: *reusable_config_test_flags ? "--test_tag_filters=-integration-test,-fix-windows" - rbe_min: - <<: *minimum_supported_version - <<: *reusable_config - name: "RBE: Ubuntu, minimum Bazel" - platform: rbe_ubuntu2404 - build_flags: - <<: *reusable_config_build_flags - # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1, - # which prevents cc toolchain autodetection from working correctly - # on Bazel 5.4 and earlier. To workaround this, manually specify the - # build kite cc toolchain. - ? "--extra_toolchains=@buildkite_config//config:cc-toolchain" - ? "--build_tag_filters=-docs" - test_flags: - <<: *reusable_config_test_flags - ? "--test_tag_filters=-integration-test,-acceptance-test,-docs" - # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1, - # which prevents cc toolchain autodetection from working correctly - # on Bazel 5.4 and earlier. To workaround this, manually specify the - # build kite cc toolchain. - ? "--extra_toolchains=@buildkite_config//config:cc-toolchain" - rbe: - <<: *reusable_config - name: "RBE: Ubuntu" - platform: rbe_ubuntu2404 - # TODO @aignas 2024-12-11: get the RBE working in CI for bazel 8.0 - # See https://github.com/bazelbuild/rules_python/issues/2499 - bazel: 8.x - test_flags: - <<: *reusable_config_test_flags - ? "--test_tag_filters=-integration-test,-acceptance-test" - ? "--extra_toolchains=@buildkite_config//config:cc-toolchain" - - integration_test_build_file_generation_ubuntu_minimum_supported_workspace: - <<: *minimum_supported_version - <<: *reusable_build_test_all - <<: *common_workspace_flags_min_bazel - name: "examples/build_file_generation: Ubuntu, workspace, minimum Bazel" - working_directory: examples/build_file_generation - platform: ubuntu2204 - integration_test_build_file_generation_ubuntu_workspace: - <<: *reusable_build_test_all - <<: *common_workspace_flags - name: "examples/build_file_generation: Ubuntu, workspace" - working_directory: examples/build_file_generation - platform: ubuntu2204 - integration_test_build_file_generation_debian_workspace: - <<: *reusable_build_test_all - <<: *common_workspace_flags - name: "examples/build_file_generation: Debian, workspace" - working_directory: examples/build_file_generation - platform: debian13 - integration_test_build_file_generation_macos_workspace: - <<: *reusable_build_test_all - <<: *common_workspace_flags - name: "examples/build_file_generation: macOS, workspace" - working_directory: examples/build_file_generation - platform: macos_arm64 integration_test_build_file_generation_windows_workspace: <<: *reusable_build_test_all <<: *common_workspace_flags @@ -384,16 +207,6 @@ tasks: working_directory: examples/build_file_generation platform: windows - integration_test_bzlmod_ubuntu_vendor: - <<: *reusable_build_test_all - name: "examples/bzlmod: bazel vendor" - working_directory: examples/bzlmod - platform: ubuntu2204 - shell_commands: - - "bazel vendor --vendor_dir=./vendor //..." - - "bazel build --vendor_dir=./vendor //..." - - "rm -rf ./vendor" - integration_test_bzlmod_windows_no_symlinks: <<: *reusable_build_test_all name: "examples/bzlmod: Windows, without symlinks" @@ -412,28 +225,6 @@ tasks: - "--keep_going" - "--action_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" - "--test_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" - - integration_test_multi_python_versions_ubuntu_workspace: - <<: *reusable_build_test_all - <<: *common_workspace_flags - <<: *coverage_targets_example_multi_python - name: "examples/multi_python_versions: Ubuntu, workspace" - working_directory: examples/multi_python_versions - platform: ubuntu2204 - integration_test_multi_python_versions_debian_workspace: - <<: *reusable_build_test_all - <<: *common_workspace_flags - <<: *coverage_targets_example_multi_python - name: "examples/multi_python_versions: Debian, workspace" - working_directory: examples/multi_python_versions - platform: debian13 - integration_test_multi_python_versions_macos_workspace: - <<: *reusable_build_test_all - <<: *common_workspace_flags - <<: *coverage_targets_example_multi_python - name: "examples/multi_python_versions: MacOS, workspace" - working_directory: examples/multi_python_versions - platform: macos_arm64 integration_test_multi_python_versions_windows_workspace: <<: *reusable_build_test_all <<: *common_workspace_flags @@ -441,87 +232,11 @@ tasks: name: "examples/multi_python_versions: Windows, workspace" working_directory: examples/multi_python_versions platform: windows - - integration_test_pip_parse_ubuntu_min_workspace: - <<: *minimum_supported_version - <<: *common_workspace_flags_min_bazel - <<: *reusable_build_test_all - name: "examples/pip_parse: Ubuntu, workspace, minimum supported Bazel version" - working_directory: examples/pip_parse - platform: ubuntu2204 - integration_test_pip_parse_ubuntu_min_bzlmod: - <<: *minimum_supported_version - <<: *reusable_build_test_all - name: "examples/pip_parse: Ubuntu, bzlmod, minimum supported Bazel version" - working_directory: examples/pip_parse - platform: ubuntu2204 - bazel: 7.x - integration_test_pip_parse_ubuntu: - <<: *reusable_build_test_all - name: "examples/pip_parse: Ubuntu" - working_directory: examples/pip_parse - platform: ubuntu2204 - integration_test_pip_parse_debian: - <<: *reusable_build_test_all - name: "examples/pip_parse: Debian" - working_directory: examples/pip_parse - platform: debian13 - integration_test_pip_parse_macos: - <<: *reusable_build_test_all - name: "examples/pip_parse: MacOS" - working_directory: examples/pip_parse - platform: macos_arm64 integration_test_pip_parse_windows: <<: *reusable_build_test_all name: "examples/pip_parse: Windows" working_directory: examples/pip_parse platform: windows - - integration_test_pip_parse_vendored_ubuntu_min_workspace: - <<: *minimum_supported_version - <<: *common_workspace_flags_min_bazel - <<: *reusable_build_test_all - name: "examples/pip_parse_vendored: Ubuntu, workspace, minimum Bazel" - working_directory: examples/pip_parse_vendored - platform: ubuntu2204 - integration_test_pip_parse_vendored_ubuntu: - <<: *reusable_build_test_all - <<: *common_workspace_flags - name: "examples/pip_parse_vendored: Ubuntu" - working_directory: examples/pip_parse_vendored - platform: ubuntu2204 - integration_test_pip_parse_vendored_debian: - <<: *reusable_build_test_all - <<: *common_workspace_flags - name: "examples/pip_parse_vendored: Debian" - working_directory: examples/pip_parse_vendored - platform: debian13 - integration_test_pip_parse_vendored_macos: - <<: *reusable_build_test_all - <<: *common_workspace_flags - name: "examples/pip_parse_vendored: MacOS" - working_directory: examples/pip_parse_vendored - platform: macos_arm64 - # We don't run pip_parse_vendored under Windows as the file checked in is - # generated from a repository rule containing OS-specific rendered paths. - - integration_test_bazelinbazel_ubuntu: - <<: *common_bazelinbazel_config - name: "tests/integration bazel-in-bazel: Ubuntu" - platform: ubuntu2204 - integration_test_bazelinbazel_debian: - <<: *common_bazelinbazel_config - name: "tests/integration bazel-in-bazel: Debian" - platform: debian13 - # The bazelinbazel tests were disabled on Mac to save CI jobs slots, and - # have bitrotted a bit. For now, just run a subset of what we're most - # interested in. - integration_test_bazelinbazel_macos: - <<: *common_bazelinbazel_config - name: "tests/integration bazel-in-bazel: macOS (subset)" - platform: macos_arm64 - build_targets: ["//tests/integration:subset"] - test_targets: ["//tests/integration:subset"] # The bazelinbazel tests were disabled on Windows to save CI jobs slots, and # have bitrotted a bit. For now, just run a subset of what we're most # interested in. @@ -531,61 +246,6 @@ tasks: platform: windows build_targets: ["//tests/integration:subset"] test_targets: ["//tests/integration:subset"] - - integration_test_compile_pip_requirements_ubuntu: - <<: *reusable_build_test_all - name: "compile_pip_requirements: Ubuntu" - working_directory: tests/integration/compile_pip_requirements - platform: ubuntu2204 - shell_commands: - # Make a change to the locked requirements and then assert that //:requirements.update does the - # right thing. - - "echo '' > requirements_lock.txt" - - "! git diff --exit-code" - - "bazel run //:requirements.update" - - "git diff --exit-code" - # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the - # right thing. - - "echo '' > requirements_lock_linux.txt" - - "! git diff --exit-code" - - "bazel run //:os_specific_requirements.update" - - "git diff --exit-code" - integration_test_compile_pip_requirements_debian: - <<: *reusable_build_test_all - name: "compile_pip_requirements: Debian" - working_directory: tests/integration/compile_pip_requirements - platform: debian13 - shell_commands: - # Make a change to the locked requirements and then assert that //:requirements.update does the - # right thing. - - "echo '' > requirements_lock.txt" - - "! git diff --exit-code" - - "bazel run //:requirements.update" - - "git diff --exit-code" - # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the - # right thing. - - "echo '' > requirements_lock_linux.txt" - - "! git diff --exit-code" - - "bazel run //:os_specific_requirements.update" - - "git diff --exit-code" - integration_test_compile_pip_requirements_macos: - <<: *reusable_build_test_all - name: "compile_pip_requirements: MacOS" - working_directory: tests/integration/compile_pip_requirements - platform: macos_arm64 - shell_commands: - # Make a change to the locked requirements and then assert that //:requirements.update does the - # right thing. - - "echo '' > requirements_lock.txt" - - "! git diff --exit-code" - - "bazel run //:requirements.update" - - "git diff --exit-code" - # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the - # right thing. - - "echo '' > requirements_lock_darwin.txt" - - "! git diff --exit-code" - - "bazel run //:os_specific_requirements.update" - - "git diff --exit-code" integration_test_compile_pip_requirements_windows: <<: *reusable_build_test_all name: "compile_pip_requirements: Windows" @@ -604,46 +264,6 @@ tasks: - "! git diff --exit-code" - "bazel run //:os_specific_requirements.update" - "git diff --exit-code" - - integration_compile_pip_requirements_test_from_external_repo_ubuntu_min_workspace: - <<: *minimum_supported_version - <<: *common_workspace_flags_min_bazel - name: "compile_pip_requirements_test_from_external_repo: Ubuntu, workspace, minimum Bazel" - working_directory: tests/integration/compile_pip_requirements_test_from_external_repo - platform: ubuntu2204 - shell_commands: - # Assert that @compile_pip_requirements//:requirements_test does the right thing. - - "bazel test @compile_pip_requirements//..." - integration_compile_pip_requirements_test_from_external_repo_ubuntu_min_bzlmod: - <<: *minimum_supported_version - name: "compile_pip_requirements_test_from_external_repo: Ubuntu, bzlmod, minimum Bazel" - working_directory: tests/integration/compile_pip_requirements_test_from_external_repo - platform: ubuntu2204 - bazel: 7.x - shell_commands: - # Assert that @compile_pip_requirements//:requirements_test does the right thing. - - "bazel test @compile_pip_requirements//..." - integration_compile_pip_requirements_test_from_external_repo_ubuntu: - name: "compile_pip_requirements_test_from_external_repo: Ubuntu" - working_directory: tests/integration/compile_pip_requirements_test_from_external_repo - platform: ubuntu2204 - shell_commands: - # Assert that @compile_pip_requirements//:requirements_test does the right thing. - - "bazel test @compile_pip_requirements//..." - integration_compile_pip_requirements_test_from_external_repo_debian: - name: "compile_pip_requirements_test_from_external_repo: Debian" - working_directory: tests/integration/compile_pip_requirements_test_from_external_repo - platform: debian13 - shell_commands: - # Assert that @compile_pip_requirements//:requirements_test does the right thing. - - "bazel test @compile_pip_requirements//..." - integration_compile_pip_requirements_test_from_external_repo_macos: - name: "compile_pip_requirements_test_from_external_repo: macOS" - working_directory: tests/integration/compile_pip_requirements_test_from_external_repo - platform: macos_arm64 - shell_commands: - # Assert that @compile_pip_requirements//:requirements_test does the right thing. - - "bazel test @compile_pip_requirements//..." integration_compile_pip_requirements_test_from_external_repo_windows: name: "compile_pip_requirements_test_from_external_repo: Windows" working_directory: tests/integration/compile_pip_requirements_test_from_external_repo From 644e4bed31b94853b8e046947dfa0e2e2a7510ab Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:24:28 +0900 Subject: [PATCH 04/19] Revert "ci: temporarily run only the Windows jobs" This reverts commit 0a739bf4e1d8fc177b34383f31e6121f87ee851e. --- .bazelci/presubmit.yml | 382 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 381 insertions(+), 1 deletion(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 52680fee0e..d7be592422 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -151,7 +151,7 @@ tasks: # LINT.IfChange(bcr_test) bcr_test: name: "BCR: Bazel {bazel}" - platform: windows + platform: ${{ platform }} working_directory: examples/bzlmod bazel: ${{ bazel }} build_flags: @@ -165,8 +165,116 @@ tasks: # LINT.ThenChange(//.bcr/presubmit.yml:tasks) # LINT.IfChange(gazelle_bcr) + gazelle_bcr_ubuntu: + <<: *gazelle_common_bcr + name: "Gazelle: BCR, Bazel {bazel}" + platform: ubuntu2204 + gazelle_bcr_debian: + <<: *gazelle_common_bcr + name: "Gazelle: BCR, Bazel {bazel}" + platform: debian13 + gazelle_bcr_macos_arm64: + <<: *gazelle_common_bcr + name: "Gazelle: BCR, Bazel {bazel}" + platform: macos_arm64 # LINT.ThenChange(//.bcr/gazelle/presubmit.yml:matrix) + gazelle_extension_min: + <<: *common_workspace_flags_min_bazel + <<: *minimum_supported_version + name: "Gazelle: workspace, minimum supported Bazel version" + platform: ubuntu2204 + build_targets: ["//..."] + test_targets: ["//..."] + working_directory: gazelle + gazelle_extension_workspace: + <<: *common_workspace_flags + name: "Gazelle: workspace" + platform: ubuntu2204 + build_targets: ["//..."] + test_targets: ["//..."] + working_directory: gazelle + gazelle_extension: + name: "Gazelle: default settings" + platform: ubuntu2204 + build_targets: ["//..."] + test_targets: ["//..."] + working_directory: gazelle + + sphinxdocs_ubuntu: + name: "Sphinxdocs: Ubuntu" + working_directory: "sphinxdocs" + platform: ubuntu2204 + shell_commands: + - "../.bazelci/isolate.sh sphinxdocs" + build_targets: + - "//..." + test_targets: + - "//..." + sphinxdocs_mac: + name: "Sphinxdocs: Mac" + working_directory: "sphinxdocs" + platform: macos_arm64 + shell_commands: + - "../.bazelci/isolate.sh sphinxdocs" + build_targets: + - "//..." + test_targets: + - "//..." + + ubuntu_min_workspace: + <<: *minimum_supported_version + <<: *reusable_config + <<: *common_workspace_flags_min_bazel + name: "Default: Ubuntu, workspace, minimum Bazel" + platform: ubuntu2204 + + ubuntu_min_bzlmod: + <<: *minimum_supported_version + <<: *reusable_config + name: "Default: Ubuntu, bzlmod, minimum Bazel" + platform: ubuntu2204 + bazel: 7.x + ubuntu: + <<: *reusable_config + <<: *coverage_targets_bootstrap + name: "Default: Ubuntu, Bazel {bazel}" + platform: ubuntu2204 + bazel: ${{ bazel }} + ubuntu_upcoming: + <<: *reusable_config + name: "Default: Ubuntu, upcoming Bazel" + platform: ubuntu2204 + bazel: last_rc + # This is an advisory job; doesn't block merges + # RCs may have regressions, so don't fail our CI on them + soft_fail: + - exit_status: 1 + - exit_status: 3 + ubuntu_rolling: + name: "Default: Ubuntu, rolling Bazel" + platform: ubuntu2204 + bazel: rolling + # This is an advisory job; doesn't block merges + soft_fail: + - exit_status: 1 + - exit_status: 3 + test_targets: + - "--" + - "//tests/..." + test_flags: + <<: *reusable_config_test_flags + ? "--keep_going" + ubuntu_workspace: + <<: *reusable_config + <<: *common_workspace_flags + name: "Default: Ubuntu, workspace" + platform: ubuntu2204 + mac_workspace: + <<: *reusable_config + <<: *common_workspace_flags + name: "Default: Mac, workspace" + platform: macos_arm64 windows_workspace: <<: *reusable_config <<: *common_workspace_flags @@ -192,6 +300,17 @@ tasks: ? "--noenable_bzlmod" ? "--enable_workspace" ? "--test_tag_filters=-integration-test" + + debian: + <<: *reusable_config + name: "Default: Debian, Bazel {bazel}" + platform: debian13 + bazel: ${{ bazel }} + macos_arm64: + <<: *reusable_config + name: "Default: MacOS, Bazel {bazel}" + platform: macos_arm64 + bazel: ${{ bazel }} windows: <<: *reusable_config name: "Default: Windows, Bazel {bazel}" @@ -200,6 +319,64 @@ tasks: test_flags: <<: *reusable_config_test_flags ? "--test_tag_filters=-integration-test,-fix-windows" + rbe_min: + <<: *minimum_supported_version + <<: *reusable_config + name: "RBE: Ubuntu, minimum Bazel" + platform: rbe_ubuntu2404 + build_flags: + <<: *reusable_config_build_flags + # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1, + # which prevents cc toolchain autodetection from working correctly + # on Bazel 5.4 and earlier. To workaround this, manually specify the + # build kite cc toolchain. + ? "--extra_toolchains=@buildkite_config//config:cc-toolchain" + ? "--build_tag_filters=-docs" + test_flags: + <<: *reusable_config_test_flags + ? "--test_tag_filters=-integration-test,-acceptance-test,-docs" + # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1, + # which prevents cc toolchain autodetection from working correctly + # on Bazel 5.4 and earlier. To workaround this, manually specify the + # build kite cc toolchain. + ? "--extra_toolchains=@buildkite_config//config:cc-toolchain" + rbe: + <<: *reusable_config + name: "RBE: Ubuntu" + platform: rbe_ubuntu2404 + # TODO @aignas 2024-12-11: get the RBE working in CI for bazel 8.0 + # See https://github.com/bazelbuild/rules_python/issues/2499 + bazel: 8.x + test_flags: + <<: *reusable_config_test_flags + ? "--test_tag_filters=-integration-test,-acceptance-test" + ? "--extra_toolchains=@buildkite_config//config:cc-toolchain" + + integration_test_build_file_generation_ubuntu_minimum_supported_workspace: + <<: *minimum_supported_version + <<: *reusable_build_test_all + <<: *common_workspace_flags_min_bazel + name: "examples/build_file_generation: Ubuntu, workspace, minimum Bazel" + working_directory: examples/build_file_generation + platform: ubuntu2204 + integration_test_build_file_generation_ubuntu_workspace: + <<: *reusable_build_test_all + <<: *common_workspace_flags + name: "examples/build_file_generation: Ubuntu, workspace" + working_directory: examples/build_file_generation + platform: ubuntu2204 + integration_test_build_file_generation_debian_workspace: + <<: *reusable_build_test_all + <<: *common_workspace_flags + name: "examples/build_file_generation: Debian, workspace" + working_directory: examples/build_file_generation + platform: debian13 + integration_test_build_file_generation_macos_workspace: + <<: *reusable_build_test_all + <<: *common_workspace_flags + name: "examples/build_file_generation: macOS, workspace" + working_directory: examples/build_file_generation + platform: macos_arm64 integration_test_build_file_generation_windows_workspace: <<: *reusable_build_test_all <<: *common_workspace_flags @@ -207,6 +384,16 @@ tasks: working_directory: examples/build_file_generation platform: windows + integration_test_bzlmod_ubuntu_vendor: + <<: *reusable_build_test_all + name: "examples/bzlmod: bazel vendor" + working_directory: examples/bzlmod + platform: ubuntu2204 + shell_commands: + - "bazel vendor --vendor_dir=./vendor //..." + - "bazel build --vendor_dir=./vendor //..." + - "rm -rf ./vendor" + integration_test_bzlmod_windows_no_symlinks: <<: *reusable_build_test_all name: "examples/bzlmod: Windows, without symlinks" @@ -225,6 +412,28 @@ tasks: - "--keep_going" - "--action_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" - "--test_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" + + integration_test_multi_python_versions_ubuntu_workspace: + <<: *reusable_build_test_all + <<: *common_workspace_flags + <<: *coverage_targets_example_multi_python + name: "examples/multi_python_versions: Ubuntu, workspace" + working_directory: examples/multi_python_versions + platform: ubuntu2204 + integration_test_multi_python_versions_debian_workspace: + <<: *reusable_build_test_all + <<: *common_workspace_flags + <<: *coverage_targets_example_multi_python + name: "examples/multi_python_versions: Debian, workspace" + working_directory: examples/multi_python_versions + platform: debian13 + integration_test_multi_python_versions_macos_workspace: + <<: *reusable_build_test_all + <<: *common_workspace_flags + <<: *coverage_targets_example_multi_python + name: "examples/multi_python_versions: MacOS, workspace" + working_directory: examples/multi_python_versions + platform: macos_arm64 integration_test_multi_python_versions_windows_workspace: <<: *reusable_build_test_all <<: *common_workspace_flags @@ -232,11 +441,87 @@ tasks: name: "examples/multi_python_versions: Windows, workspace" working_directory: examples/multi_python_versions platform: windows + + integration_test_pip_parse_ubuntu_min_workspace: + <<: *minimum_supported_version + <<: *common_workspace_flags_min_bazel + <<: *reusable_build_test_all + name: "examples/pip_parse: Ubuntu, workspace, minimum supported Bazel version" + working_directory: examples/pip_parse + platform: ubuntu2204 + integration_test_pip_parse_ubuntu_min_bzlmod: + <<: *minimum_supported_version + <<: *reusable_build_test_all + name: "examples/pip_parse: Ubuntu, bzlmod, minimum supported Bazel version" + working_directory: examples/pip_parse + platform: ubuntu2204 + bazel: 7.x + integration_test_pip_parse_ubuntu: + <<: *reusable_build_test_all + name: "examples/pip_parse: Ubuntu" + working_directory: examples/pip_parse + platform: ubuntu2204 + integration_test_pip_parse_debian: + <<: *reusable_build_test_all + name: "examples/pip_parse: Debian" + working_directory: examples/pip_parse + platform: debian13 + integration_test_pip_parse_macos: + <<: *reusable_build_test_all + name: "examples/pip_parse: MacOS" + working_directory: examples/pip_parse + platform: macos_arm64 integration_test_pip_parse_windows: <<: *reusable_build_test_all name: "examples/pip_parse: Windows" working_directory: examples/pip_parse platform: windows + + integration_test_pip_parse_vendored_ubuntu_min_workspace: + <<: *minimum_supported_version + <<: *common_workspace_flags_min_bazel + <<: *reusable_build_test_all + name: "examples/pip_parse_vendored: Ubuntu, workspace, minimum Bazel" + working_directory: examples/pip_parse_vendored + platform: ubuntu2204 + integration_test_pip_parse_vendored_ubuntu: + <<: *reusable_build_test_all + <<: *common_workspace_flags + name: "examples/pip_parse_vendored: Ubuntu" + working_directory: examples/pip_parse_vendored + platform: ubuntu2204 + integration_test_pip_parse_vendored_debian: + <<: *reusable_build_test_all + <<: *common_workspace_flags + name: "examples/pip_parse_vendored: Debian" + working_directory: examples/pip_parse_vendored + platform: debian13 + integration_test_pip_parse_vendored_macos: + <<: *reusable_build_test_all + <<: *common_workspace_flags + name: "examples/pip_parse_vendored: MacOS" + working_directory: examples/pip_parse_vendored + platform: macos_arm64 + # We don't run pip_parse_vendored under Windows as the file checked in is + # generated from a repository rule containing OS-specific rendered paths. + + integration_test_bazelinbazel_ubuntu: + <<: *common_bazelinbazel_config + name: "tests/integration bazel-in-bazel: Ubuntu" + platform: ubuntu2204 + integration_test_bazelinbazel_debian: + <<: *common_bazelinbazel_config + name: "tests/integration bazel-in-bazel: Debian" + platform: debian13 + # The bazelinbazel tests were disabled on Mac to save CI jobs slots, and + # have bitrotted a bit. For now, just run a subset of what we're most + # interested in. + integration_test_bazelinbazel_macos: + <<: *common_bazelinbazel_config + name: "tests/integration bazel-in-bazel: macOS (subset)" + platform: macos_arm64 + build_targets: ["//tests/integration:subset"] + test_targets: ["//tests/integration:subset"] # The bazelinbazel tests were disabled on Windows to save CI jobs slots, and # have bitrotted a bit. For now, just run a subset of what we're most # interested in. @@ -246,6 +531,61 @@ tasks: platform: windows build_targets: ["//tests/integration:subset"] test_targets: ["//tests/integration:subset"] + + integration_test_compile_pip_requirements_ubuntu: + <<: *reusable_build_test_all + name: "compile_pip_requirements: Ubuntu" + working_directory: tests/integration/compile_pip_requirements + platform: ubuntu2204 + shell_commands: + # Make a change to the locked requirements and then assert that //:requirements.update does the + # right thing. + - "echo '' > requirements_lock.txt" + - "! git diff --exit-code" + - "bazel run //:requirements.update" + - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_linux.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" + integration_test_compile_pip_requirements_debian: + <<: *reusable_build_test_all + name: "compile_pip_requirements: Debian" + working_directory: tests/integration/compile_pip_requirements + platform: debian13 + shell_commands: + # Make a change to the locked requirements and then assert that //:requirements.update does the + # right thing. + - "echo '' > requirements_lock.txt" + - "! git diff --exit-code" + - "bazel run //:requirements.update" + - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_linux.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" + integration_test_compile_pip_requirements_macos: + <<: *reusable_build_test_all + name: "compile_pip_requirements: MacOS" + working_directory: tests/integration/compile_pip_requirements + platform: macos_arm64 + shell_commands: + # Make a change to the locked requirements and then assert that //:requirements.update does the + # right thing. + - "echo '' > requirements_lock.txt" + - "! git diff --exit-code" + - "bazel run //:requirements.update" + - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_darwin.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_windows: <<: *reusable_build_test_all name: "compile_pip_requirements: Windows" @@ -264,6 +604,46 @@ tasks: - "! git diff --exit-code" - "bazel run //:os_specific_requirements.update" - "git diff --exit-code" + + integration_compile_pip_requirements_test_from_external_repo_ubuntu_min_workspace: + <<: *minimum_supported_version + <<: *common_workspace_flags_min_bazel + name: "compile_pip_requirements_test_from_external_repo: Ubuntu, workspace, minimum Bazel" + working_directory: tests/integration/compile_pip_requirements_test_from_external_repo + platform: ubuntu2204 + shell_commands: + # Assert that @compile_pip_requirements//:requirements_test does the right thing. + - "bazel test @compile_pip_requirements//..." + integration_compile_pip_requirements_test_from_external_repo_ubuntu_min_bzlmod: + <<: *minimum_supported_version + name: "compile_pip_requirements_test_from_external_repo: Ubuntu, bzlmod, minimum Bazel" + working_directory: tests/integration/compile_pip_requirements_test_from_external_repo + platform: ubuntu2204 + bazel: 7.x + shell_commands: + # Assert that @compile_pip_requirements//:requirements_test does the right thing. + - "bazel test @compile_pip_requirements//..." + integration_compile_pip_requirements_test_from_external_repo_ubuntu: + name: "compile_pip_requirements_test_from_external_repo: Ubuntu" + working_directory: tests/integration/compile_pip_requirements_test_from_external_repo + platform: ubuntu2204 + shell_commands: + # Assert that @compile_pip_requirements//:requirements_test does the right thing. + - "bazel test @compile_pip_requirements//..." + integration_compile_pip_requirements_test_from_external_repo_debian: + name: "compile_pip_requirements_test_from_external_repo: Debian" + working_directory: tests/integration/compile_pip_requirements_test_from_external_repo + platform: debian13 + shell_commands: + # Assert that @compile_pip_requirements//:requirements_test does the right thing. + - "bazel test @compile_pip_requirements//..." + integration_compile_pip_requirements_test_from_external_repo_macos: + name: "compile_pip_requirements_test_from_external_repo: macOS" + working_directory: tests/integration/compile_pip_requirements_test_from_external_repo + platform: macos_arm64 + shell_commands: + # Assert that @compile_pip_requirements//:requirements_test does the right thing. + - "bazel test @compile_pip_requirements//..." integration_compile_pip_requirements_test_from_external_repo_windows: name: "compile_pip_requirements_test_from_external_repo: Windows" working_directory: tests/integration/compile_pip_requirements_test_from_external_repo From 656966d0309bc6af76b4cf722fea6c085430e992 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:26:18 +0900 Subject: [PATCH 05/19] test: check the venv without symlinks in Python for Windows paths On Windows, the venv paths are Windows paths, which the bash checks couldn't handle. --- tests/bootstrap_impls/bin.py | 2 ++ .../run_binary_without_symlinks_test.sh | 25 +++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/bootstrap_impls/bin.py b/tests/bootstrap_impls/bin.py index 7ebe0c047e..5913d92413 100644 --- a/tests/bootstrap_impls/bin.py +++ b/tests/bootstrap_impls/bin.py @@ -37,4 +37,6 @@ def handle(signum, _frame): print("sys.flags.safe_path:", sys.flags.safe_path) print("file:", __file__) print("sys.executable:", sys.executable) +print("sys.executable is symlink:", os.path.islink(sys.executable)) +print("sys.prefix:", sys.prefix) print("sys._base_executable:", getattr(sys, "_base_executable", None)) diff --git a/tests/bootstrap_impls/run_binary_without_symlinks_test.sh b/tests/bootstrap_impls/run_binary_without_symlinks_test.sh index b056eda29b..6c1c775e7f 100755 --- a/tests/bootstrap_impls/run_binary_without_symlinks_test.sh +++ b/tests/bootstrap_impls/run_binary_without_symlinks_test.sh @@ -30,27 +30,32 @@ function expect_match() { fi } +# Converts a Windows path to a Unix one under MSYS, so that bash can use it. +function to_unix() { + if command -v cygpath >/dev/null; then + cygpath -u "$1" + else + echo "$1" + fi +} + # Use hard links, junctions or copies instead of symlinks for the venv that # is created at runtime, like on Windows without the privilege to create # symlinks. venvs_root=$(mktemp -d) actual=$(RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1 RULES_PYTHON_EXTRACT_ROOT=$venvs_root $bin) expect_match "Hello" "$actual" -expect_match "sys.executable:.*$venvs_root" "$actual" - -venv_python=$(echo "$actual" | grep "^sys.executable:" | cut -d' ' -f2-) -if [[ -L "$venv_python" ]]; then - echo "expected the venv interpreter not to be a symlink: $venv_python" - touch EXPECTATION_FAILED -fi +# Match the name only, because the path is a Windows path on Windows. +expect_match "sys.executable:.*$(basename "$venvs_root")" "$actual" +expect_match "sys.executable is symlink: False" "$actual" # Without RULES_PYTHON_EXTRACT_ROOT, the venv is removed after the run, which # must work for read-only copies of Bazel outputs too. actual=$(RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1 $bin) expect_match "Hello" "$actual" -venv_python=$(echo "$actual" | grep "^sys.executable:" | cut -d' ' -f2-) -venv=$(dirname "$(dirname "$venv_python")") -if [[ -e "$venv" ]]; then +expect_match "sys.executable is symlink: False" "$actual" +venv=$(to_unix "$(echo "$actual" | grep "^sys.prefix:" | cut -d' ' -f2-)") +if [[ -z "$venv" || -e "$venv" ]]; then echo "expected the venv to be removed: $venv" touch EXPECTATION_FAILED fi From fdc4cb1826652563811ed704e6af977b04d476ff Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:29:09 +0900 Subject: [PATCH 06/19] ci: fix the flags of the Windows job without symlinks Use the `? "--flag"` keys that presubmit_test requires, and download the remote cache outputs, because without symlinks Bazel creates a junction in the runfiles tree for an output that isn't downloaded yet. --- .bazelci/presubmit.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index d7be592422..ec9b198ce8 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -405,13 +405,18 @@ tasks: # check that binaries work without the privilege to create symlinks. - "echo startup --nowindows_enable_symlinks >> .bazelrc" build_flags: - - "--keep_going" + ? "--keep_going" # The CI user may be able to create symlinks, so force the fallback. - - "--action_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" + ? "--action_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" + # Without symlinks, Bazel creates a junction instead of a copy in the + # runfiles tree for an output that isn't downloaded from the remote + # cache yet, and a junction to a file doesn't work. + ? "--remote_download_all" test_flags: - - "--keep_going" - - "--action_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" - - "--test_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" + ? "--keep_going" + ? "--action_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" + ? "--remote_download_all" + ? "--test_env=RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1" integration_test_multi_python_versions_ubuntu_workspace: <<: *reusable_build_test_all From e5d01cc8403b76a8b64049d72b75ccf65972db38 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 15 Nov 2026 23:39:30 +0900 Subject: [PATCH 07/19] fix(windows): keep the read-only attribute of hard linked outputs The fallback for when symlinks can't be created is now Windows only. Windows refuses to delete a read-only file and the attribute is shared by all the hard links of a file, so removing the venv made the hard linked Bazel outputs writable. The hard links are now removed first, and the attribute is restored on their originals. --- python/private/python_bootstrap_template.txt | 86 ++++++++++++++----- tests/bootstrap_impls/BUILD.bazel | 10 ++- .../run_binary_without_symlinks_test.sh | 14 ++- 3 files changed, 80 insertions(+), 30 deletions(-) diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 7d6d0a6656..f8a036fdd2 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -453,11 +453,7 @@ print(site.getsitepackages(["{venv_src}"])[-1]) if os.path.exists(runfiles_venv_lib): _symlink_tree(link_from=venv_lib, link_to=runfiles_venv_lib) - if IS_WINDOWS or not _SYMLINKS_SUPPORTED: - if python_home is None: - # Without a symlink, Python can't find home from the venv - # interpreter, so it has to be in pyvenv.cfg. - python_home = dirname(python_exe_actual) + if IS_WINDOWS: print_verbose("create_venv: pyvenv.cfg home: ", python_home) venv_pyvenv_cfg = join(venv, "pyvenv.cfg") with open(venv_pyvenv_cfg, "w") as fp: @@ -620,14 +616,20 @@ def _run_execv(python_program, argv, env): print_verbose("pyprog head:" + str(f.read(50))) raise -# Set to False once creating a symlink fails, e.g. on Windows without the +# Windows only: set to False once creating a symlink fails, e.g. without the # privilege to create symlinks, so that it isn't attempted for every file. # Tests set the environment variable to use the fallback even when symlinks # can be created. -_SYMLINKS_SUPPORTED = not os.environ.get("RULES_PYTHON_TESTING_DISABLE_SYMLINKS") +_SYMLINKS_SUPPORTED = not ( + IS_WINDOWS and os.environ.get("RULES_PYTHON_TESTING_DISABLE_SYMLINKS") +) + +# Windows only: (link, original) pairs of the hard links that were created, +# so that they can be removed without changing their originals. +_HARD_LINKS = [] def _symlink_exist_ok(*, from_, to): - """Creates a symlink, or a hard link, junction or copy if that's not possible.""" + """Creates a symlink, or on Windows, a hard link, junction or copy if that's not possible.""" global _SYMLINKS_SUPPORTED # On Windows, symlinks have to be told whether they're # pointing to a file or directory. Python is supposed to auto-detect @@ -640,41 +642,79 @@ def _symlink_exist_ok(*, from_, to): except FileExistsError: return except OSError as e: + if not IS_WINDOWS: + raise print_verbose("symlink failed, falling back to links or copies:", e) _SYMLINKS_SUPPORTED = False if os.path.lexists(from_): return if is_dir: - _link_dir(from_, to) + _link_dir_windows(from_, to) else: - _link_file(from_, to) + _link_file_windows(from_, to) -def _link_file(from_, to): +def _link_file_windows(from_, to): # Hard links don't need any privileges, but only work within the same # volume, so fall back to copying the file. try: os.link(to, from_) + _HARD_LINKS.append((from_, to)) except OSError as e: print_verbose("hard link failed, copying:", to, "error:", e) shutil.copy2(to, from_) -def _link_dir(from_, to): - if IS_WINDOWS: +def _link_dir_windows(from_, to): + try: + # Junctions don't need any privileges, but need an absolute target. + import _winapi + _winapi.CreateJunction(abspath(to), from_) + except (ImportError, AttributeError, OSError) as e: + print_verbose("junction failed, copying:", to, "error:", e) + shutil.copytree(to, from_) + +def _remove_hard_links_windows(): + """Removes the hard links, keeping the read-only attribute of their originals. + + Windows refuses to delete a read-only file, such as a Bazel output, and the + attribute is shared by all the hard links of a file, so it is cleared to + remove the link and then restored on the original. + + An alternative is to delete the link with + FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE via ctypes, which doesn't + change the original at all, but needs Windows 10 1809 or later and more + Windows specific code. + """ + import stat + for link, original in _HARD_LINKS: try: - # Junctions don't need any privileges, but need an absolute target. - import _winapi - _winapi.CreateJunction(abspath(to), from_) - return - except (ImportError, AttributeError, OSError) as e: - print_verbose("junction failed, copying:", to, "error:", e) - shutil.copytree(to, from_) + os.remove(link) + continue + except FileNotFoundError: + continue + except PermissionError: + pass + try: + mode = os.stat(original).st_mode + os.chmod(link, mode | stat.S_IWRITE) + try: + os.remove(link) + finally: + os.chmod(original, mode) + except OSError as e: + print_verbose("cleanup: failed to remove hard link:", link, "error:", e) + del _HARD_LINKS[:] def _rmtree(path): - """Removes a directory, including read-only files, e.g. copies of outputs.""" + if not IS_WINDOWS: + shutil.rmtree(path, True) + return + + _remove_hard_links_windows() + def _on_error(func, error_path, _): - # Files copied or hard linked from Bazel outputs are read-only and - # Windows refuses to delete them. + # The remaining files are copies, which are read-only if their originals + # are, and Windows refuses to delete read-only files. try: os.chmod(error_path, 0o777) func(error_path) diff --git a/tests/bootstrap_impls/BUILD.bazel b/tests/bootstrap_impls/BUILD.bazel index d95dbd0be1..d17cfd8f28 100644 --- a/tests/bootstrap_impls/BUILD.bazel +++ b/tests/bootstrap_impls/BUILD.bazel @@ -17,7 +17,7 @@ load("//python:py_test.bzl", "py_test") load("//python:pytest_test.bzl", "pytest_test") load("//tests/support:py_reconfig.bzl", "py_reconfig_binary", "py_reconfig_test") load("//tests/support:sh_py_run_test.bzl", "sh_py_run_test") -load("//tests/support:support.bzl", "BAZEL_8_OR_LATER", "SUPPORTS_BOOTSTRAP_SCRIPT") +load("//tests/support:support.bzl", "SUPPORTS_BOOTSTRAP_SCRIPT") load(":venv_relative_path_tests.bzl", "relative_path_test_suite") py_reconfig_binary( @@ -78,9 +78,11 @@ sh_py_run_test( bootstrap_impl = "system_python", py_src = "bin.py", sh_src = "run_binary_without_symlinks_test.sh", - # Bazel 7 doesn't create a full venv for bootstrap_impl=system_python. - target_compatible_with = BAZEL_8_OR_LATER, - venvs_use_declare_symlink = "no", + # The fallback for when symlinks can't be created is Windows only. + target_compatible_with = select({ + "@platforms//os:windows": [], + "//conditions:default": ["@platforms//:incompatible"], + }), ) sh_py_run_test( diff --git a/tests/bootstrap_impls/run_binary_without_symlinks_test.sh b/tests/bootstrap_impls/run_binary_without_symlinks_test.sh index 6c1c775e7f..acac79f5bf 100755 --- a/tests/bootstrap_impls/run_binary_without_symlinks_test.sh +++ b/tests/bootstrap_impls/run_binary_without_symlinks_test.sh @@ -40,8 +40,7 @@ function to_unix() { } # Use hard links, junctions or copies instead of symlinks for the venv that -# is created at runtime, like on Windows without the privilege to create -# symlinks. +# is created at runtime, like without the privilege to create symlinks. venvs_root=$(mktemp -d) actual=$(RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1 RULES_PYTHON_EXTRACT_ROOT=$venvs_root $bin) expect_match "Hello" "$actual" @@ -50,7 +49,10 @@ expect_match "sys.executable:.*$(basename "$venvs_root")" "$actual" expect_match "sys.executable is symlink: False" "$actual" # Without RULES_PYTHON_EXTRACT_ROOT, the venv is removed after the run, which -# must work for read-only copies of Bazel outputs too. +# must work for read-only Bazel outputs that are hard linked or copied, and must +# not make the hard linked outputs writable. +runfiles=$(to_unix "${RUNFILES_DIR:-$TEST_SRCDIR}") +read_only_before=$(find "$runfiles" -type f ! -writable | sort) actual=$(RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1 $bin) expect_match "Hello" "$actual" expect_match "sys.executable is symlink: False" "$actual" @@ -59,6 +61,12 @@ if [[ -z "$venv" || -e "$venv" ]]; then echo "expected the venv to be removed: $venv" touch EXPECTATION_FAILED fi +read_only_after=$(find "$runfiles" -type f ! -writable | sort) +if [[ "$read_only_before" != "$read_only_after" ]]; then + echo "expected the read-only runfiles to be unchanged" + diff <(echo "$read_only_before") <(echo "$read_only_after") + touch EXPECTATION_FAILED +fi # Exit if any of the expects failed [[ ! -e EXPECTATION_FAILED ]] From 22b7ee5d402e7abe8bd3b8aab7032e0cb16277f7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 15 Nov 2026 23:48:39 +0900 Subject: [PATCH 08/19] fix(windows): delete read-only files without clearing the attribute Instead of restoring the read-only attribute of hard linked Bazel outputs after clearing it, delete the read-only files of the venv with FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE, which doesn't change the other hard links of a file. It needs Windows 10 1809 or Windows Server 2019, so older versions still require symlinks. --- news/3792.fixed.md | 5 +- python/private/python_bootstrap_template.txt | 102 +++++++++++-------- 2 files changed, 64 insertions(+), 43 deletions(-) diff --git a/news/3792.fixed.md b/news/3792.fixed.md index 01cdc76da4..08dfda52ca 100644 --- a/news/3792.fixed.md +++ b/news/3792.fixed.md @@ -1,4 +1,5 @@ (binaries/tests) (Windows) `py_binary` and `py_test` no longer need the -privilege to create symlinks to run. If creating a symlink fails, the venv -that is created at runtime uses hard links, junctions or copies instead +privilege to create symlinks to run on Windows 10 1809, Windows Server 2019 +or later. If creating a symlink fails, the venv that is created at runtime +uses hard links, junctions or copies instead ([#3792](https://github.com/bazel-contrib/rules_python/issues/3792)). diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index f8a036fdd2..394085ba7e 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -624,9 +624,10 @@ _SYMLINKS_SUPPORTED = not ( IS_WINDOWS and os.environ.get("RULES_PYTHON_TESTING_DISABLE_SYMLINKS") ) -# Windows only: (link, original) pairs of the hard links that were created, -# so that they can be removed without changing their originals. -_HARD_LINKS = [] +# The build of Windows 10 1809 and Windows Server 2019, which added +# FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE that the fallback for when +# symlinks can't be created needs to clean up. +_WINDOWS_FALLBACK_MIN_BUILD = 17763 def _symlink_exist_ok(*, from_, to): """Creates a symlink, or on Windows, a hard link, junction or copy if that's not possible.""" @@ -644,6 +645,11 @@ def _symlink_exist_ok(*, from_, to): except OSError as e: if not IS_WINDOWS: raise + if sys.getwindowsversion().build < _WINDOWS_FALLBACK_MIN_BUILD: + raise OSError( + "Creating symlinks failed, which is required before Windows 10 " + "1809 and Windows Server 2019: {}".format(e) + ) from e print_verbose("symlink failed, falling back to links or copies:", e) _SYMLINKS_SUPPORTED = False @@ -659,7 +665,6 @@ def _link_file_windows(from_, to): # volume, so fall back to copying the file. try: os.link(to, from_) - _HARD_LINKS.append((from_, to)) except OSError as e: print_verbose("hard link failed, copying:", to, "error:", e) shutil.copy2(to, from_) @@ -673,53 +678,68 @@ def _link_dir_windows(from_, to): print_verbose("junction failed, copying:", to, "error:", e) shutil.copytree(to, from_) -def _remove_hard_links_windows(): - """Removes the hard links, keeping the read-only attribute of their originals. +def _remove_read_only_file_windows(path): + """Removes a read-only file without changing its read-only attribute. - Windows refuses to delete a read-only file, such as a Bazel output, and the - attribute is shared by all the hard links of a file, so it is cleared to - remove the link and then restored on the original. - - An alternative is to delete the link with - FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE via ctypes, which doesn't - change the original at all, but needs Windows 10 1809 or later and more - Windows specific code. + Windows refuses to delete a read-only file, such as a hard link to or a + copy of a Bazel output. Clearing the attribute would also clear it for the + other hard links of the file, i.e. the Bazel output, so the file is deleted + with FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE instead. """ - import stat - for link, original in _HARD_LINKS: - try: - os.remove(link) - continue - except FileNotFoundError: - continue - except PermissionError: - pass - try: - mode = os.stat(original).st_mode - os.chmod(link, mode | stat.S_IWRITE) - try: - os.remove(link) - finally: - os.chmod(original, mode) - except OSError as e: - print_verbose("cleanup: failed to remove hard link:", link, "error:", e) - del _HARD_LINKS[:] + import ctypes + from ctypes import wintypes + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.CreateFileW.restype = wintypes.HANDLE + kernel32.CreateFileW.argtypes = [ + wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, wintypes.LPVOID, + wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE, + ] + kernel32.SetFileInformationByHandle.restype = wintypes.BOOL + kernel32.SetFileInformationByHandle.argtypes = [ + wintypes.HANDLE, ctypes.c_int, wintypes.LPVOID, wintypes.DWORD, + ] + kernel32.CloseHandle.argtypes = [wintypes.HANDLE] + + DELETE = 0x00010000 + FILE_SHARE_READ_WRITE_DELETE = 0x7 + OPEN_EXISTING = 3 + FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000 + FILE_DISPOSITION_INFO_EX = 21 + FILE_DISPOSITION_FLAG_DELETE = 0x1 + FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE = 0x10 + INVALID_HANDLE_VALUE = wintypes.HANDLE(-1).value + + handle = kernel32.CreateFileW( + path, DELETE, FILE_SHARE_READ_WRITE_DELETE, None, OPEN_EXISTING, + FILE_FLAG_OPEN_REPARSE_POINT, None, + ) + if handle == INVALID_HANDLE_VALUE: + raise ctypes.WinError(ctypes.get_last_error()) + try: + flags = wintypes.DWORD( + FILE_DISPOSITION_FLAG_DELETE | FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE + ) + if not kernel32.SetFileInformationByHandle( + handle, FILE_DISPOSITION_INFO_EX, ctypes.byref(flags), ctypes.sizeof(flags) + ): + raise ctypes.WinError(ctypes.get_last_error()) + finally: + # The file is deleted when the handle is closed. + kernel32.CloseHandle(handle) def _rmtree(path): if not IS_WINDOWS: shutil.rmtree(path, True) return - _remove_hard_links_windows() - def _on_error(func, error_path, _): - # The remaining files are copies, which are read-only if their originals - # are, and Windows refuses to delete read-only files. + if func not in (os.unlink, os.remove): + return try: - os.chmod(error_path, 0o777) - func(error_path) - except OSError: - pass + _remove_read_only_file_windows(error_path) + except OSError as e: + print_verbose("cleanup: failed to remove:", error_path, "error:", e) if sys.version_info >= (3, 12): shutil.rmtree(path, onexc=_on_error) From 52e600e19effed5bd4a0aa3fb4c75ea017419c45 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:52:34 +0900 Subject: [PATCH 09/19] refactor(windows): define the symlink fallback globals with the other globals Move them next to the other top-level globals and use a lower case name for the one that changes at runtime. --- python/private/python_bootstrap_template.txt | 32 ++++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 394085ba7e..6a5f1a52b4 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -116,6 +116,19 @@ IS_WINDOWS = os.name == "nt" BIN_DIR_NAME = "bin" if not IS_WINDOWS else "Scripts" LIB_DIR_NAME = "lib" if not IS_WINDOWS else "Lib" +# Windows only: set to False once creating a symlink fails, e.g. without the +# privilege to create symlinks, so that it isn't attempted for every file. +# Tests set the environment variable to use the fallback even when symlinks +# can be created. +_symlinks_supported = not ( + IS_WINDOWS and os.environ.get("RULES_PYTHON_TESTING_DISABLE_SYMLINKS") +) + +# The build of Windows 10 1809 and Windows Server 2019, which added +# FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE that the fallback for when +# symlinks can't be created needs to clean up. +_WINDOWS_FALLBACK_MIN_BUILD = 17763 + # Windows APIs can be picky about slashes depending on the context, # so convert to backslashes to avoid any issues. if IS_WINDOWS: @@ -616,27 +629,14 @@ def _run_execv(python_program, argv, env): print_verbose("pyprog head:" + str(f.read(50))) raise -# Windows only: set to False once creating a symlink fails, e.g. without the -# privilege to create symlinks, so that it isn't attempted for every file. -# Tests set the environment variable to use the fallback even when symlinks -# can be created. -_SYMLINKS_SUPPORTED = not ( - IS_WINDOWS and os.environ.get("RULES_PYTHON_TESTING_DISABLE_SYMLINKS") -) - -# The build of Windows 10 1809 and Windows Server 2019, which added -# FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE that the fallback for when -# symlinks can't be created needs to clean up. -_WINDOWS_FALLBACK_MIN_BUILD = 17763 - def _symlink_exist_ok(*, from_, to): """Creates a symlink, or on Windows, a hard link, junction or copy if that's not possible.""" - global _SYMLINKS_SUPPORTED + global _symlinks_supported # On Windows, symlinks have to be told whether they're # pointing to a file or directory. Python is supposed to auto-detect # this, but in practice, this doesn't reliably happen. is_dir = os.path.isdir(to) - if _SYMLINKS_SUPPORTED: + if _symlinks_supported: try: os.symlink(to, from_, target_is_directory=is_dir) return @@ -651,7 +651,7 @@ def _symlink_exist_ok(*, from_, to): "1809 and Windows Server 2019: {}".format(e) ) from e print_verbose("symlink failed, falling back to links or copies:", e) - _SYMLINKS_SUPPORTED = False + _symlinks_supported = False if os.path.lexists(from_): return From 29437d65ba095c1fa8fd01130181338082693872 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:52:48 +0900 Subject: [PATCH 10/19] fix(windows): link to the target of a symlink in the fallback When Bazel could create the runfiles symlinks, but the venv falls back to hard links, os.link() hard linked the symlink itself, so sys.executable was still a symlink. Resolve the target first. --- python/private/python_bootstrap_template.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 6a5f1a52b4..a0df5f9b06 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -655,6 +655,10 @@ def _symlink_exist_ok(*, from_, to): if os.path.lexists(from_): return + # The target can itself be a symlink, e.g. when Bazel could create the + # runfiles symlinks. A hard link to a symlink is also a symlink, so link to + # the file or directory that it points to instead. + to = os.path.realpath(to) if is_dir: _link_dir_windows(from_, to) else: From c8a319a0e7acd860ef3682a292606294ff235e1a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:52:59 +0900 Subject: [PATCH 11/19] fix(windows): don't copy over a venv entry that another process created When processes share RULES_PYTHON_EXTRACT_ROOT, another one can create the entry between the existence check and creating the link, which made the fallback copy onto it. Also only catch OSError from _winapi.CreateJunction, which every supported Python on Windows has. --- python/private/python_bootstrap_template.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index a0df5f9b06..fbf2234669 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -669,16 +669,22 @@ def _link_file_windows(from_, to): # volume, so fall back to copying the file. try: os.link(to, from_) + # Another process sharing RULES_PYTHON_EXTRACT_ROOT created it first. + except FileExistsError: + return except OSError as e: print_verbose("hard link failed, copying:", to, "error:", e) shutil.copy2(to, from_) def _link_dir_windows(from_, to): + import _winapi try: # Junctions don't need any privileges, but need an absolute target. - import _winapi - _winapi.CreateJunction(abspath(to), from_) - except (ImportError, AttributeError, OSError) as e: + _winapi.CreateJunction(to, from_) + # Another process sharing RULES_PYTHON_EXTRACT_ROOT created it first. + except FileExistsError: + return + except OSError as e: print_verbose("junction failed, copying:", to, "error:", e) shutil.copytree(to, from_) From 0c8698705362a4481dae019617567c8f491b275a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:53:13 +0900 Subject: [PATCH 12/19] refactor(windows): set up the kernel32 prototypes only once They were set up again for every read-only file that is removed. Also declare the return type of CloseHandle. --- python/private/python_bootstrap_template.txt | 30 +++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index fbf2234669..31f59fc851 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -688,14 +688,14 @@ def _link_dir_windows(from_, to): print_verbose("junction failed, copying:", to, "error:", e) shutil.copytree(to, from_) -def _remove_read_only_file_windows(path): - """Removes a read-only file without changing its read-only attribute. +_kernel32 = None + +def _get_kernel32(): + """Returns kernel32 with the prototypes that cleaning up the venv needs.""" + global _kernel32 + if _kernel32 is not None: + return _kernel32 - Windows refuses to delete a read-only file, such as a hard link to or a - copy of a Bazel output. Clearing the attribute would also clear it for the - other hard links of the file, i.e. the Bazel output, so the file is deleted - with FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE instead. - """ import ctypes from ctypes import wintypes @@ -709,7 +709,23 @@ def _remove_read_only_file_windows(path): kernel32.SetFileInformationByHandle.argtypes = [ wintypes.HANDLE, ctypes.c_int, wintypes.LPVOID, wintypes.DWORD, ] + kernel32.CloseHandle.restype = wintypes.BOOL kernel32.CloseHandle.argtypes = [wintypes.HANDLE] + _kernel32 = kernel32 + return kernel32 + +def _remove_read_only_file_windows(path): + """Removes a read-only file without changing its read-only attribute. + + Windows refuses to delete a read-only file, such as a hard link to or a + copy of a Bazel output. Clearing the attribute would also clear it for the + other hard links of the file, i.e. the Bazel output, so the file is deleted + with FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE instead. + """ + import ctypes + from ctypes import wintypes + + kernel32 = _get_kernel32() DELETE = 0x00010000 FILE_SHARE_READ_WRITE_DELETE = 0x7 From a9d0b11eb666451ff756ef94166d126e417b3cd6 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:53:23 +0900 Subject: [PATCH 13/19] fix(windows): remove a dangling junction left by a cached venv The fallback creates junctions for directories, and islink() is False for them, so a dangling one wasn't removed and makedirs() failed. --- python/private/python_bootstrap_template.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 31f59fc851..4699b71472 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -454,11 +454,10 @@ print(site.getsitepackages(["{venv_src}"])[-1]) link_to = normpath(join(runfiles_root, link_to_rf_path)) venv_parent = dirname(venv_abs_path) # A cached venv can leave this parent as a dangling link to an old EXTRACT_ROOT. - if os.path.islink(venv_parent): - try: - os.stat(venv_parent) - except FileNotFoundError: - os.unlink(venv_parent) + # islink() is False for a junction on Windows, but lexists() doesn't + # follow it, and unlink() removes it. + if os.path.lexists(venv_parent) and not os.path.exists(venv_parent): + os.unlink(venv_parent) os.makedirs(venv_parent, exist_ok=True) _symlink_exist_ok(from_=venv_abs_path, to=link_to) From 8a43e513a3fe1ac1689054ba8f449b776058d518 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:53:51 +0900 Subject: [PATCH 14/19] test(windows): make the test without symlinks check what it claims Remove the CR of the Windows output, so that the check that the venv was removed doesn't always pass, follow the runfiles symlinks when looking for read-only outputs and require that there are some, create the venvs under TEST_TMPDIR, and quote the paths. --- .../run_binary_without_symlinks_test.sh | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/bootstrap_impls/run_binary_without_symlinks_test.sh b/tests/bootstrap_impls/run_binary_without_symlinks_test.sh index acac79f5bf..fc8fea9002 100755 --- a/tests/bootstrap_impls/run_binary_without_symlinks_test.sh +++ b/tests/bootstrap_impls/run_binary_without_symlinks_test.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # --- begin runfiles.bash initialization v3 --- # Copy-pasted from the Bazel Bash runfiles library v3. set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash @@ -10,12 +11,16 @@ source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ # --- end runfiles.bash initialization v3 --- set +e -bin=$(rlocation $BIN_RLOCATION) +bin=$(rlocation "$BIN_RLOCATION") if [[ -z "$bin" ]]; then echo "Unable to locate test binary: $BIN_RLOCATION" exit 1 fi +# Checks that the output matches a pattern. +# +# Modifies: +# Creates EXPECTATION_FAILED in the working directory on mismatch. function expect_match() { local expected_pattern=$1 local actual=$2 @@ -32,17 +37,14 @@ function expect_match() { # Converts a Windows path to a Unix one under MSYS, so that bash can use it. function to_unix() { - if command -v cygpath >/dev/null; then - cygpath -u "$1" - else - echo "$1" - fi + cygpath --unix "$1" } # Use hard links, junctions or copies instead of symlinks for the venv that # is created at runtime, like without the privilege to create symlinks. -venvs_root=$(mktemp -d) -actual=$(RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1 RULES_PYTHON_EXTRACT_ROOT=$venvs_root $bin) +# tr: remove the CR of the CRLF line endings of the output on Windows. +venvs_root=$(mktemp --directory "$(to_unix "$TEST_TMPDIR")/venvs_root.XXXXXX") +actual=$(RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1 RULES_PYTHON_EXTRACT_ROOT="$venvs_root" "$bin" | tr --delete '\r') expect_match "Hello" "$actual" # Match the name only, because the path is a Windows path on Windows. expect_match "sys.executable:.*$(basename "$venvs_root")" "$actual" @@ -52,16 +54,21 @@ expect_match "sys.executable is symlink: False" "$actual" # must work for read-only Bazel outputs that are hard linked or copied, and must # not make the hard linked outputs writable. runfiles=$(to_unix "${RUNFILES_DIR:-$TEST_SRCDIR}") -read_only_before=$(find "$runfiles" -type f ! -writable | sort) -actual=$(RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1 $bin) +# -L: follow the runfiles symlinks to the Bazel outputs. +read_only_before=$(find -L "$runfiles" -type f ! -writable | sort) +if [[ -z "$read_only_before" ]]; then + echo "expected read-only files in the runfiles: $runfiles" + touch EXPECTATION_FAILED +fi +actual=$(RULES_PYTHON_TESTING_DISABLE_SYMLINKS=1 "$bin" | tr --delete '\r') expect_match "Hello" "$actual" expect_match "sys.executable is symlink: False" "$actual" -venv=$(to_unix "$(echo "$actual" | grep "^sys.prefix:" | cut -d' ' -f2-)") +venv=$(to_unix "$(echo "$actual" | grep "^sys.prefix:" | cut --delimiter=' ' --fields=2-)") if [[ -z "$venv" || -e "$venv" ]]; then echo "expected the venv to be removed: $venv" touch EXPECTATION_FAILED fi -read_only_after=$(find "$runfiles" -type f ! -writable | sort) +read_only_after=$(find -L "$runfiles" -type f ! -writable | sort) if [[ "$read_only_before" != "$read_only_after" ]]; then echo "expected the read-only runfiles to be unchanged" diff <(echo "$read_only_before") <(echo "$read_only_after") From e073dab9d07623f3eaf74620318699d3f6f02de7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:54:00 +0900 Subject: [PATCH 15/19] test: use the Windows constraint directly --- tests/bootstrap_impls/BUILD.bazel | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/bootstrap_impls/BUILD.bazel b/tests/bootstrap_impls/BUILD.bazel index d17cfd8f28..34db553dbd 100644 --- a/tests/bootstrap_impls/BUILD.bazel +++ b/tests/bootstrap_impls/BUILD.bazel @@ -79,10 +79,7 @@ sh_py_run_test( py_src = "bin.py", sh_src = "run_binary_without_symlinks_test.sh", # The fallback for when symlinks can't be created is Windows only. - target_compatible_with = select({ - "@platforms//os:windows": [], - "//conditions:default": ["@platforms//:incompatible"], - }), + target_compatible_with = ["@platforms//os:windows"], ) sh_py_run_test( From aadea93ccb1e6dcf38569ed4fa66ab4eaf335b43 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:54:32 +0900 Subject: [PATCH 16/19] docs: link the rules in the news entry --- news/3792.fixed.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/news/3792.fixed.md b/news/3792.fixed.md index 08dfda52ca..5a7dfa3795 100644 --- a/news/3792.fixed.md +++ b/news/3792.fixed.md @@ -1,5 +1,5 @@ -(binaries/tests) (Windows) `py_binary` and `py_test` no longer need the -privilege to create symlinks to run on Windows 10 1809, Windows Server 2019 -or later. If creating a symlink fails, the venv that is created at runtime +(binaries/tests) (Windows) {obj}`py_binary` and {obj}`py_test` no longer need +the privilege to create symlinks to run on Windows 10 1809, Windows Server +2019 or later. If creating a symlink fails, the venv that is created at runtime uses hard links, junctions or copies instead ([#3792](https://github.com/bazel-contrib/rules_python/issues/3792)). From 4fb4ca964600c0adbb8e4444c5b8f20089e06d8b Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:02:47 +0900 Subject: [PATCH 17/19] test(windows): check that a dangling junction of a cached venv is removed Check the behavior of lexists(), exists() and unlink() for a dangling junction, which the bootstrap relies on, and run the cached venv test with the fallback for when symlinks can't be created. --- .../cached_venv_dangling_symlink_test.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py b/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py index fc59b23017..89be2edbea 100644 --- a/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py +++ b/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py @@ -43,7 +43,39 @@ def _load_bootstrap() -> dict: class CachedVenvDanglingSymlinkTest(unittest.TestCase): def test_replaces_dangling_package_link_before_creating_child_link(self) -> None: + self._test_replaces_dangling_package_link(symlinks_supported=True) + + @unittest.skipUnless(os.name == "nt", "junctions are Windows only") + def test_replaces_dangling_package_junction_before_creating_child_link( + self, + ) -> None: + self._test_replaces_dangling_package_link(symlinks_supported=False) + + @unittest.skipUnless(os.name == "nt", "junctions are Windows only") + def test_dangling_junction_is_detected_and_removed(self) -> None: + # The bootstrap relies on these to remove a dangling junction, which + # islink() doesn't detect. + import _winapi + + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + target = root / "target" + target.mkdir() + junction = root / "junction" + _winapi.CreateJunction(str(target), str(junction)) # pyrefly: ignore[missing-attribute] + target.rmdir() + + self.assertFalse(os.path.islink(junction)) + self.assertTrue(os.path.lexists(junction)) + self.assertFalse(os.path.exists(junction)) + os.unlink(junction) + self.assertFalse(os.path.lexists(junction)) + + def _test_replaces_dangling_package_link(self, *, symlinks_supported) -> None: bootstrap = _load_bootstrap() + # False uses the fallback for when symlinks can't be created, which + # creates a junction for a directory. + bootstrap["_symlinks_supported"] = symlinks_supported with tempfile.TemporaryDirectory() as temp: root = pathlib.Path(temp) runfiles = root / "runfiles" @@ -78,7 +110,9 @@ def test_replaces_dangling_package_link_before_creating_child_link(self) -> None package_init = package / "__init__.py" self.assertTrue(package.is_dir()) self.assertFalse(package.is_symlink()) - self.assertTrue(package_init.is_symlink()) + self.assertFalse(getattr(os.path, "isjunction", lambda _: False)(package)) + # The fallback creates a hard link for a file. + self.assertEqual(package_init.is_symlink(), symlinks_supported) self.assertTrue(os.path.samefile(package_init, new_package / "__init__.py")) From 688af3e76b81a7c841ecafc9f4fa04ea83715bd5 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:02:49 +0900 Subject: [PATCH 18/19] docs(windows): note that copying a directory can still race --- python/private/python_bootstrap_template.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 4699b71472..72bf62ddad 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -685,6 +685,10 @@ def _link_dir_windows(from_, to): return except OSError as e: print_verbose("junction failed, copying:", to, "error:", e) + # This can still fail with FileExistsError if another process sharing + # RULES_PYTHON_EXTRACT_ROOT creates the directory while copying. Both + # creating the junction and a concurrent run have to fail for that, so + # this isn't handled. shutil.copytree(to, from_) _kernel32 = None From 52dc7ef329eccb640a61d5fa2cd6feee5da06ad6 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:39:38 +0900 Subject: [PATCH 19/19] test(windows): don't hard link to the running interpreter in the cached venv test Windows can't delete a hard link to a running executable, so cleaning up the temporary directory failed after the fallback hard linked sys.executable into the venv. The venv isn't run, so link to a stand-in file instead. --- .../bootstrap_impls/cached_venv_dangling_symlink_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py b/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py index 89be2edbea..363736d0a0 100644 --- a/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py +++ b/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py @@ -82,10 +82,16 @@ def _test_replaces_dangling_package_link(self, *, symlinks_supported) -> None: old_package = runfiles / "old/annotated_types" old_package.mkdir(parents=True) (old_package / "__init__.py").touch() + # The venv isn't run, and Windows can't delete a hard link to the + # running interpreter, e.g. when cleaning up the temporary + # directory, so link to a stand-in instead. + python = root / "python/python.exe" + python.parent.mkdir() + python.touch() bootstrap.update( EXTRACT_ROOT=str(root / "cache"), PYTHON_BINARY="_main/_process.venv/Scripts/python.exe", - PYTHON_BINARY_ACTUAL=sys.executable, + PYTHON_BINARY_ACTUAL=str(python), RESOLVE_PYTHON_BINARY_AT_RUNTIME=False, RUNTIME_VENV_SYMLINKS={ "Lib/site-packages/annotated_types": "old/annotated_types",