Add support for @ENV as an exe preference

.github/workflows/tests.yml

@@ -64,12 +64,15 @@ jobs:
         with:
           python-version: ${{ matrix.pyversion }}
 
+      # Always build PyCall (and hence Conda) so that Conda's root env directory exists;
+      # otherwise Conda fails to precompile on cached runners, which can cause errors.
+      # Use a temporary environment because `--project=test` has no manifest on Julia
+      # versions without workspace support (< 1.12).
       - name: Build PyCall
-        if: ${{ matrix.pythonexe == 'python' }}
         run: |
-          julia --project=test -e 'import Pkg; Pkg.build("PyCall")'
+          julia -e 'import Pkg; Pkg.activate(temp=true); Pkg.add("PyCall"); Pkg.build("PyCall")'
         env:
-          PYTHON: ${{ steps.setup-python.outputs.python-path }}
+          PYTHON: ${{ case(matrix.pythonexe == 'python', steps.setup-python.outputs.python-path, 'python') }}
 
       - name: Run tests
         uses: julia-actions/julia-runtest@v1

docs/src/pythoncall.md

@@ -309,6 +309,15 @@ special values:
 - `@CondaPkg`: Use Python from CondaPkg (the default).
 - `@PyCall`: Use the same Python as PyCall. [See here](@ref faq-pycall).
 - `@venv`: Use Python from a `.venv` virtual environment in the current active project.
+- `@ENV`: This value should only be set as the `exe` preference. When set,
+  PythonCall will look up the `exe` value explicitly from the
+  `JULIA_PYTHONCALL_EXE` environment variable, which _must_ be a valid path or
+  command name (see below). The advantage of setting `@ENV` as a preference is
+  that CondaPkg.jl will not be loaded, which saves on load time, while you still
+  keep the ability to change Python environments based on environment
+  variables. This can be helpful when using shared Julia environments on a
+  cluster so that users can easily switch Python environments without modifying
+  the depot preferences.
 
 Otherwise, the value is interpreted as:
 - An absolute path to a Python executable.

src/C/context.jl

@@ -160,9 +160,16 @@ function init_context()
             else
                 exe_path = abspath(exe_path, "bin", "python")::String
             end
-        elseif startswith(exe_path, "@")
+        elseif startswith(exe_path, "@") && exe_path != "@ENV"
             error("invalid exe: $exe_path")
         else
+            if exe_path == "@ENV"
+                exe_path = get(ENV, "JULIA_PYTHONCALL_EXE", "")
+                if isempty(exe_path)
+                    error("PythonCall's `exe` preference is set to `@ENV`, but JULIA_PYTHONCALL_EXE is not set.")
+                end
+            end
+
             # Otherwise we use the Python specified
             CTX.which = :unknown
             if isabspath(exe_path)

test/C.jl

@@ -16,3 +16,31 @@
         @test PythonCall.python_version().major == 3
     end
 end
+
+@testitem "exe = @ENV" begin
+    mktempdir() do dir
+        # A throwaway environment prepended to the load path overrides the `exe`
+        # preference for a child process while still resolving PythonCall from ours.
+        write(
+            joinpath(dir, "Project.toml"),
+            "[extras]\nPythonCall = \"6099a3de-0909-46bc-b1f4-468b9a2dfc0d\"\n",
+        )
+        write(joinpath(dir, "LocalPreferences.toml"), "[PythonCall]\nexe = \"@ENV\"\n")
+        sep = Sys.iswindows() ? ";" : ":"
+        loadpath = join([dir, Base.active_project(), "@stdlib"], sep)
+        exe = PythonCall.python_executable_path()
+        code = "using PythonCall; print(PythonCall.python_executable_path())"
+        cmd = `$(Base.julia_cmd()) --startup-file=no -e $code`
+
+        # variable set: it is used as the executable
+        withset = addenv(cmd, "JULIA_LOAD_PATH" => loadpath, "JULIA_PYTHONCALL_EXE" => exe)
+        @test readchomp(withset) == exe
+
+        # variable unset: fails at load time with a clear message
+        unset = addenv(cmd, "JULIA_LOAD_PATH" => loadpath, "JULIA_PYTHONCALL_EXE" => nothing)
+        err = IOBuffer()
+        proc = run(pipeline(ignorestatus(unset), stdout = devnull, stderr = err))
+        @test !success(proc)
+        @test occursin("JULIA_PYTHONCALL_EXE is not set", String(take!(err)))
+    end
+end