From 639464583745c344c8130df25e9c23bd360d82ca Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 15 Nov 2026 20:22:38 -0800 Subject: [PATCH 1/2] refactor(wheel): convert py_wheel_dist build tool to Rust Copying stamped wheel outputs into a dist directory currently spawns a Python interpreter per `py_wheel_dist` target. Add `//crates/py_wheel_dist:py_wheel_dist_lib`, register `py_wheel_dist` on `shrubbery`, and route `PyWheelDistDir` through `get_shrubbery_tool`. `get_shrubbery_tool` gains `files_to_run` and `is_rust` so rules can pass the resolved tool straight to `ctx.actions.run` and branch on which implementation resolved. --- crates/py_wheel_dist/BUILD.bazel | 20 ++ crates/py_wheel_dist/src/lib.rs | 181 +++++++++++++ crates/shrubbery/BUILD.bazel | 1 + crates/shrubbery/src/lib.rs | 3 + python/BUILD.bazel | 2 + python/packaging.bzl | 11 +- .../toolchains/shrubbery_toolchain.bzl | 19 +- tests/py_wheel_dist/BUILD.bazel | 15 ++ tests/py_wheel_dist/py_wheel_dist_test.rs | 245 ++++++++++++++++++ tests/shrubbery/shrubbery_test.rs | 2 +- 10 files changed, 493 insertions(+), 6 deletions(-) create mode 100644 crates/py_wheel_dist/BUILD.bazel create mode 100644 crates/py_wheel_dist/src/lib.rs create mode 100644 tests/py_wheel_dist/BUILD.bazel create mode 100644 tests/py_wheel_dist/py_wheel_dist_test.rs diff --git a/crates/py_wheel_dist/BUILD.bazel b/crates/py_wheel_dist/BUILD.bazel new file mode 100644 index 0000000000..18af91ed29 --- /dev/null +++ b/crates/py_wheel_dist/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_rust//rust:defs.bzl", "rust_library") +load("//python/private:distribution_filegroup.bzl", "distribution_filegroup") # buildifier: disable=bzl-visibility + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_library( + name = "py_wheel_dist_lib", + srcs = ["src/lib.rs"], + edition = "2024", + deps = [ + "@crates//:argfile", + "@crates//:lexopt", + ], +) + +distribution_filegroup( + name = "distribution", +) diff --git a/crates/py_wheel_dist/src/lib.rs b/crates/py_wheel_dist/src/lib.rs new file mode 100644 index 0000000000..ecf7e3ebf1 --- /dev/null +++ b/crates/py_wheel_dist/src/lib.rs @@ -0,0 +1,181 @@ +//! Library for generating the output directory for `py_wheel_dist`. +//! +//! Usage: +//! `py_wheel_dist --wheel --name_file --output ` +//! +//! Reads the canonical wheel filename from `--name_file` (UTF-8, whitespace-trimmed), +//! creates the `--output` directory, and copies `--wheel` to `/`. + +use std::ffi::OsString; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +/// Validates and trims the wheel filename read from `--name_file`. +/// +/// Ensures the trimmed name is non-empty and a valid single file basename without +/// path separators (`/`, `\`) or parent directory traversal (`..`). +pub fn validate_wheel_name(raw_name: &str) -> io::Result<&str> { + let trimmed = raw_name.trim(); + if trimmed.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "wheel name in --name_file is empty", + )); + } + if trimmed == "." + || trimmed == ".." + || trimmed.contains('/') + || trimmed.contains('\\') + || trimmed.contains("..") + || trimmed.contains('\0') + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("invalid wheel filename in --name_file: '{trimmed}'"), + )); + } + Ok(trimmed) +} + +/// Creates the output directory and copies the wheel file to `/`. +pub fn create_wheel_dist( + wheel_path: &Path, + name_file_path: &Path, + output_dir: &Path, +) -> io::Result { + let raw_name = fs::read_to_string(name_file_path)?; + let wheel_name = validate_wheel_name(&raw_name)?; + + fs::create_dir_all(output_dir)?; + + let destination = output_dir.join(wheel_name); + fs::copy(wheel_path, &destination)?; + Ok(destination) +} + +/// Parsed command-line options for `py_wheel_dist`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Options { + /// Path to the input `.whl` file. + pub wheel: PathBuf, + /// Path to the file containing the canonical wheel filename. + pub name_file: PathBuf, + /// Path to the output directory where the wheel should be copied. + pub output: PathBuf, +} + +fn parse_flagfile(content: &str, prefix: char) -> Vec { + content + .lines() + .filter(|line| !line.is_empty()) + .map(|line| argfile::Argument::parse_ref(line, prefix)) + .collect() +} + +fn parse_non_empty_path(parser: &mut lexopt::Parser, flag: &str) -> Result { + let val = parser.value().map_err(|e| e.to_string())?; + if val.is_empty() { + return Err(format!("empty value for {flag}")); + } + Ok(PathBuf::from(val)) +} + +/// Parses command-line arguments (excluding `argv[0]`) into [`Options`]. +/// +/// Supports `--flag value`, `--flag=value`, and `@flagfile` argument expansion. +pub fn parse_args(args: I) -> Result +where + I: IntoIterator, + T: Into, +{ + use lexopt::prelude::*; + + let raw_args: Vec = args.into_iter().map(Into::into).collect(); + if raw_args.iter().any(|arg| arg == "@") { + return Err("empty @flagfile path".to_string()); + } + let expanded = argfile::expand_args_from(raw_args.into_iter(), parse_flagfile, argfile::PREFIX) + .map_err(|e| format!("failed to read flagfile: {e}"))?; + + let mut wheel = None; + let mut name_file = None; + let mut output = None; + + let mut parser = lexopt::Parser::from_args(expanded); + while let Some(arg) = parser.next().map_err(|e| e.to_string())? { + match arg { + Short('h') | Long("help") => { + return Err( + "Usage: py_wheel_dist --wheel --name_file --output " + .to_string(), + ); + } + Long("wheel") => wheel = Some(parse_non_empty_path(&mut parser, "--wheel")?), + Long("name_file") => { + name_file = Some(parse_non_empty_path(&mut parser, "--name_file")?); + } + Long("output") => output = Some(parse_non_empty_path(&mut parser, "--output")?), + Long(other) => { + let other = other.to_owned(); + if let Some(val) = parser.optional_value() { + return Err(format!( + "unexpected argument: --{other}={}", + val.to_string_lossy() + )); + } + return Err(format!("unexpected argument: --{other}")); + } + Short(ch) => { + if let Some(val) = parser.optional_value() { + return Err(format!( + "unexpected argument: -{ch}{}", + val.to_string_lossy() + )); + } + return Err(format!("unexpected argument: -{ch}")); + } + Value(val) => { + return Err(format!("unexpected argument: {}", val.to_string_lossy())); + } + } + } + + let wheel = wheel.ok_or_else(|| "missing required argument: --wheel".to_string())?; + let name_file = + name_file.ok_or_else(|| "missing required argument: --name_file".to_string())?; + let output = output.ok_or_else(|| "missing required argument: --output".to_string())?; + + Ok(Options { + wheel, + name_file, + output, + }) +} + +/// Executes `py_wheel_dist` with the parsed [`Options`]. +pub fn run(options: &Options) -> io::Result { + create_wheel_dist(&options.wheel, &options.name_file, &options.output) +} + +/// Parses CLI arguments (excluding `argv[0]` and subcommand name), populates the wheel dist +/// directory, and returns the process exit code. +pub fn run_cli(args: I) -> ExitCode +where + I: IntoIterator, + T: Into, +{ + let options = match parse_args(args) { + Ok(opts) => opts, + Err(msg) => { + eprintln!("py_wheel_dist: error: {msg}"); + return ExitCode::FAILURE; + } + }; + if let Err(err) = run(&options) { + eprintln!("py_wheel_dist: error: {err}"); + return ExitCode::FAILURE; + } + ExitCode::SUCCESS +} diff --git a/crates/shrubbery/BUILD.bazel b/crates/shrubbery/BUILD.bazel index 314c595d86..1c37ddd5d6 100644 --- a/crates/shrubbery/BUILD.bazel +++ b/crates/shrubbery/BUILD.bazel @@ -11,6 +11,7 @@ rust_library( edition = "2024", deps = [ "//crates/exe_zip_maker:exe_zip_maker_lib", + "//crates/py_wheel_dist:py_wheel_dist_lib", "//crates/wheelmaker:wheelmaker_lib", "//crates/zip_main_maker:zip_main_maker_lib", "//crates/zipper:zipper_lib", diff --git a/crates/shrubbery/src/lib.rs b/crates/shrubbery/src/lib.rs index cd17e358ee..1ed3656642 100644 --- a/crates/shrubbery/src/lib.rs +++ b/crates/shrubbery/src/lib.rs @@ -11,6 +11,7 @@ use std::process::ExitCode; pub mod subcommands { // keep-sorted start case=no pub const EXE_ZIP_MAKER: &str = "exe_zip_maker"; + pub const PY_WHEEL_DIST: &str = "py_wheel_dist"; pub const WHEELMAKER: &str = "wheelmaker"; pub const ZIP_MAIN_MAKER: &str = "zip_main_maker"; pub const ZIPPER: &str = "zipper"; @@ -21,6 +22,7 @@ pub mod subcommands { pub const SUBCOMMANDS: &[&str] = &[ // keep-sorted start case=no subcommands::EXE_ZIP_MAKER, + subcommands::PY_WHEEL_DIST, subcommands::WHEELMAKER, subcommands::ZIP_MAIN_MAKER, subcommands::ZIPPER, @@ -66,6 +68,7 @@ where match sub_raw.as_ref() { // keep-sorted start case=no subcommands::EXE_ZIP_MAKER => exe_zip_maker_lib::run_cli(iter), + subcommands::PY_WHEEL_DIST => py_wheel_dist_lib::run_cli(iter), subcommands::WHEELMAKER => wheelmaker_lib::run_cli(iter), subcommands::ZIP_MAIN_MAKER => zip_main_maker_lib::run_cli(iter), subcommands::ZIPPER => zipper_lib::run_cli(iter), diff --git a/python/BUILD.bazel b/python/BUILD.bazel index f824bb226e..d447a467a3 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -209,7 +209,9 @@ bzl_library( "//python/private:bzlmod_enabled", "//python/private:py_package", "//python/private:py_wheel", + "//python/private:toolchain_types", "//python/private:util", + "//python/private/toolchains:shrubbery_toolchain", "@bazel_skylib//rules:native_binary", ], ) diff --git a/python/packaging.bzl b/python/packaging.bzl index 1ec1d380df..de006653cf 100644 --- a/python/packaging.bzl +++ b/python/packaging.bzl @@ -19,7 +19,9 @@ load("//python:py_binary.bzl", "py_binary") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:py_package.bzl", "py_package_lib") load("//python/private:py_wheel.bzl", _PyWheelInfo = "PyWheelInfo", _py_wheel = "py_wheel") +load("//python/private:toolchain_types.bzl", "SHRUBBERY_TOOLCHAIN_TYPE") load("//python/private:util.bzl", "copy_propagating_kwargs") +load("//python/private/toolchains:shrubbery_toolchain.bzl", "get_shrubbery_tool") # Re-export as public API PyWheelInfo = _PyWheelInfo @@ -45,12 +47,14 @@ def _py_wheel_dist_impl(ctx): args.add("--name_file", name_file) args.add("--output", out.path) + tool = get_shrubbery_tool(ctx, "py_wheel_dist", ctx.attr._copier) ctx.actions.run( mnemonic = "PyWheelDistDir", - executable = ctx.executable._copier, + executable = tool.files_to_run, + toolchain = tool.toolchain, inputs = [wheel, name_file], outputs = [out], - arguments = [args], + arguments = tool.prefix_args + [args], ) return [ DefaultInfo( @@ -84,6 +88,9 @@ This also has the advantage that stamping information is included in the wheel's default = Label("//python/private:py_wheel_dist"), ), }, + toolchains = [ + config_common.toolchain_type(SHRUBBERY_TOOLCHAIN_TYPE, mandatory = False), + ], ) def py_wheel( diff --git a/python/private/toolchains/shrubbery_toolchain.bzl b/python/private/toolchains/shrubbery_toolchain.bzl index 757d44f838..694cdc0ca3 100644 --- a/python/private/toolchains/shrubbery_toolchain.bzl +++ b/python/private/toolchains/shrubbery_toolchain.bzl @@ -61,8 +61,12 @@ def get_shrubbery_tool(ctx, subcommand, fallback): Returns: {type}`struct` with fields: * `executable`: {type}`Target` to pass to `actions_run()`. - * `prefix_args`: {type}`list[str]` `[subcommand]` when resolved from - the `shrubbery` toolchain, else `[]`. + * `files_to_run`: {type}`FilesToRunProvider` to pass to + `ctx.actions.run(executable = ...)`. + * `is_rust`: {type}`bool` True when resolved from the `shrubbery` + toolchain. + * `prefix_args`: {type}`list[str]` `[subcommand]` when `is_rust` is + True, else `[]`. * `toolchain`: {type}`Label | None` the toolchain type to pass to `actions_run()` or `ctx.actions.run()` so the action runs on the exec platform the tool was built for. `None` when falling back to @@ -70,9 +74,14 @@ def get_shrubbery_tool(ctx, subcommand, fallback): """ toolchain = ctx.toolchains[SHRUBBERY_TOOLCHAIN_TYPE] if toolchain and toolchain.shrubbery: + executable = toolchain.shrubbery return struct( # Target to pass to `actions_run()`. - executable = toolchain.shrubbery, + executable = executable, + # `FilesToRunProvider` to pass to `ctx.actions.run(executable = ...)`. + files_to_run = executable[DefaultInfo].files_to_run, + # True when resolved from the `shrubbery` toolchain. + is_rust = True, # Leading subcommand argument prepended when invoking `shrubbery`. prefix_args = [subcommand], # Toolchain type for `actions_run()` / `ctx.actions.run()`. @@ -81,6 +90,10 @@ def get_shrubbery_tool(ctx, subcommand, fallback): return struct( # Fallback tool target to pass to `actions_run()`. executable = fallback, + # `FilesToRunProvider` to pass to `ctx.actions.run(executable = ...)`. + files_to_run = fallback[DefaultInfo].files_to_run, + # False when falling back to the built-in Python or script tool. + is_rust = False, # Empty prefix when invoking the standalone fallback tool directly. prefix_args = [], # `None` so fallback tools use the default exec tools or host execution. diff --git a/tests/py_wheel_dist/BUILD.bazel b/tests/py_wheel_dist/BUILD.bazel new file mode 100644 index 0000000000..1f37536562 --- /dev/null +++ b/tests/py_wheel_dist/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_rust//rust:defs.bzl", "rust_test") + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_test( + name = "py_wheel_dist_test", + size = "small", + srcs = ["py_wheel_dist_test.rs"], + edition = "2024", + deps = [ + "//crates/py_wheel_dist:py_wheel_dist_lib", + ], +) diff --git a/tests/py_wheel_dist/py_wheel_dist_test.rs b/tests/py_wheel_dist/py_wheel_dist_test.rs new file mode 100644 index 0000000000..9a3a6456e8 --- /dev/null +++ b/tests/py_wheel_dist/py_wheel_dist_test.rs @@ -0,0 +1,245 @@ +use std::env; +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use py_wheel_dist_lib::{ + Options, create_wheel_dist, parse_args, run, run_cli, validate_wheel_name, +}; + +static COUNTER: AtomicUsize = AtomicUsize::new(0); + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn new(prefix: &str) -> Self { + let id = COUNTER.fetch_add(1, Ordering::SeqCst); + let path = env::temp_dir().join(format!( + "py_wheel_dist_test_{}_{}_{}", + prefix, + std::process::id(), + id + )); + fs::create_dir_all(&path).unwrap(); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +#[test] +fn test_parse_args_space_separated() { + let args = [ + "--wheel", + "path/to/foo.whl", + "--name_file", + "path/to/name.txt", + "--output", + "path/to/dist", + ]; + let opts = parse_args(args).unwrap(); + assert_eq!( + opts, + Options { + wheel: PathBuf::from("path/to/foo.whl"), + name_file: PathBuf::from("path/to/name.txt"), + output: PathBuf::from("path/to/dist"), + } + ); +} + +#[test] +fn test_parse_args_equals_separated() { + let args = [ + "--wheel=path/to/foo.whl", + "--name_file=path/to/name.txt", + "--output=path/to/dist", + ]; + let opts = parse_args(args).unwrap(); + assert_eq!( + opts, + Options { + wheel: PathBuf::from("path/to/foo.whl"), + name_file: PathBuf::from("path/to/name.txt"), + output: PathBuf::from("path/to/dist"), + } + ); +} + +#[test] +fn test_parse_args_flagfile() { + let tmp = TempDir::new("flagfile"); + let flagfile = tmp.path().join("args.params"); + fs::write( + &flagfile, + "--wheel\n\nmy_wheel.whl\n--name_file=my_name.txt\n\n--output\nout_dir\n", + ) + .unwrap(); + + let arg = format!("@{}", flagfile.display()); + let opts = parse_args([arg]).unwrap(); + assert_eq!( + opts, + Options { + wheel: PathBuf::from("my_wheel.whl"), + name_file: PathBuf::from("my_name.txt"), + output: PathBuf::from("out_dir"), + } + ); +} + +#[test] +fn test_parse_args_mixed_flagfile_and_overrides() { + let tmp = TempDir::new("mixed_flagfile"); + let flagfile = tmp.path().join("args.params"); + fs::write(&flagfile, "--wheel\nfirst.whl\n--name_file\nname.txt\n").unwrap(); + + let flagfile_arg = format!("@{}", flagfile.display()); + let opts = parse_args([ + flagfile_arg.as_str(), + "--wheel=override.whl", + "--output", + "final_out", + ]) + .unwrap(); + assert_eq!( + opts, + Options { + wheel: PathBuf::from("override.whl"), + name_file: PathBuf::from("name.txt"), + output: PathBuf::from("final_out"), + } + ); +} + +#[test] +fn test_parse_args_missing_required_flags() { + let err_no_wheel = parse_args(["--name_file", "a", "--output", "b"]).unwrap_err(); + assert!(err_no_wheel.contains("--wheel"), "got: {err_no_wheel}"); + + let err_no_name = parse_args(["--wheel", "a", "--output", "b"]).unwrap_err(); + assert!(err_no_name.contains("--name_file"), "got: {err_no_name}"); + + let err_no_out = parse_args(["--wheel", "a", "--name_file", "b"]).unwrap_err(); + assert!(err_no_out.contains("--output"), "got: {err_no_out}"); +} + +#[test] +fn test_parse_args_missing_flag_values_and_unknown_args() { + assert!(parse_args(["--wheel"]).is_err()); + assert!(parse_args(["--name_file"]).is_err()); + assert!(parse_args(["--output"]).is_err()); + assert!(parse_args(["--wheel="]).is_err()); + assert!(parse_args(["--name_file="]).is_err()); + assert!(parse_args(["--output="]).is_err()); + assert!(parse_args(["--unknown", "val"]).is_err()); + let bogus_err = parse_args(["--bogus-flag=123"]).unwrap_err(); + assert!( + bogus_err.contains("unexpected argument: --bogus-flag=123"), + "got: {bogus_err}" + ); + assert!(parse_args(["positional_arg"]).is_err()); + assert!(parse_args(["@"]).is_err()); + assert!(parse_args(["@nonexistent_flagfile_path_12345"]).is_err()); +} + +#[test] +fn test_validate_wheel_name() { + assert_eq!( + validate_wheel_name("example_pkg-0.0.1-py3-none-any.whl").unwrap(), + "example_pkg-0.0.1-py3-none-any.whl" + ); + assert_eq!( + validate_wheel_name(" \n\t example_pkg-1.2.3-py3-none-any.whl \r\n ").unwrap(), + "example_pkg-1.2.3-py3-none-any.whl" + ); + + assert!(validate_wheel_name("").is_err()); + assert!(validate_wheel_name(" \n\t ").is_err()); + assert!(validate_wheel_name(".").is_err()); + assert!(validate_wheel_name("..").is_err()); + assert!(validate_wheel_name("../evil.whl").is_err()); + assert!(validate_wheel_name("sub/wheel.whl").is_err()); + assert!(validate_wheel_name("sub\\wheel.whl").is_err()); +} + +#[test] +fn test_copy_wheel_into_new_and_existing_output_directory() { + let tmp = TempDir::new("copy"); + let wheel_path = tmp.path().join("input.whl"); + let name_file_path = tmp.path().join("name.txt"); + let out_dir = tmp.path().join("nested").join("dist_out"); + + let wheel_bytes = b"PK\x03\x04fake-wheel-archive-bytes"; + fs::write(&wheel_path, wheel_bytes).unwrap(); + fs::write(&name_file_path, " my_pkg-0.1.0-py3-none-any.whl\n").unwrap(); + + // 1. Copy into a new (non-existent) nested output directory. + let opts = Options { + wheel: wheel_path.clone(), + name_file: name_file_path.clone(), + output: out_dir.clone(), + }; + let copied_path = run(&opts).unwrap(); + assert_eq!(copied_path, out_dir.join("my_pkg-0.1.0-py3-none-any.whl")); + assert_eq!(fs::read(&copied_path).unwrap(), wheel_bytes); + + // 2. Copy into an already-existing output directory with updated wheel content. + let updated_bytes = b"PK\x03\x04updated-wheel-archive-bytes"; + fs::write(&wheel_path, updated_bytes).unwrap(); + fs::write(&name_file_path, "my_pkg-0.2.0-py3-none-any.whl").unwrap(); + + let second_path = create_wheel_dist(&wheel_path, &name_file_path, &out_dir).unwrap(); + assert_eq!(second_path, out_dir.join("my_pkg-0.2.0-py3-none-any.whl")); + assert_eq!(fs::read(&second_path).unwrap(), updated_bytes); + + // 3. Verify run_cli returns ExitCode::SUCCESS and ExitCode::FAILURE. + assert_eq!( + run_cli([ + OsStr::new("--wheel"), + wheel_path.as_os_str(), + OsStr::new("--name_file"), + name_file_path.as_os_str(), + OsStr::new("--output"), + out_dir.as_os_str(), + ]), + ExitCode::SUCCESS + ); + assert_eq!(run_cli(["--unknown"]), ExitCode::FAILURE); +} + +#[test] +fn test_error_handling_missing_inputs_and_invalid_name() { + let tmp = TempDir::new("errors"); + let wheel_path = tmp.path().join("input.whl"); + let name_file_path = tmp.path().join("name.txt"); + let out_dir = tmp.path().join("dist"); + + // Missing both files. + assert!(create_wheel_dist(&wheel_path, &name_file_path, &out_dir).is_err()); + + // Present name_file, missing wheel file. + fs::write(&name_file_path, "pkg-0.1.0-py3-none-any.whl").unwrap(); + assert!(create_wheel_dist(&wheel_path, &name_file_path, &out_dir).is_err()); + + // Present wheel file, empty/whitespace name_file. + fs::write(&wheel_path, b"wheel-data").unwrap(); + fs::write(&name_file_path, " \n ").unwrap(); + assert!(create_wheel_dist(&wheel_path, &name_file_path, &out_dir).is_err()); + + // Invalid traversal name in name_file. + fs::write(&name_file_path, "../escape.whl").unwrap(); + assert!(create_wheel_dist(&wheel_path, &name_file_path, &out_dir).is_err()); +} diff --git a/tests/shrubbery/shrubbery_test.rs b/tests/shrubbery/shrubbery_test.rs index 372fa34842..7127746284 100644 --- a/tests/shrubbery/shrubbery_test.rs +++ b/tests/shrubbery/shrubbery_test.rs @@ -17,7 +17,7 @@ fn temp_test_dir(prefix: &str) -> PathBuf { #[test] fn test_subcommands_registry_sorted_and_complete() { - assert_eq!(SUBCOMMANDS.len(), 4); + assert_eq!(SUBCOMMANDS.len(), 5); let mut sorted = SUBCOMMANDS.to_vec(); sorted.sort_unstable(); assert_eq!(SUBCOMMANDS, sorted.as_slice()); From d24c5359eedd50efb7d21d0c1bc5fc1c44250021 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 15 Nov 2026 20:49:55 -0800 Subject: [PATCH 2/2] py_wheel_dist: follow the shared CLI conventions Drop the wheel-name validation that the Python tool never had, use argfile's parse_fromfile directly instead of a per-crate flagfile parser, and adopt the USAGE constant, flags module, anyhow errors, and long-only arguments used by the other shrubbery subcommands. --- crates/py_wheel_dist/BUILD.bazel | 1 + crates/py_wheel_dist/src/lib.rs | 193 ++++++++++------------ tests/py_wheel_dist/py_wheel_dist_test.rs | 154 +++++++++-------- 3 files changed, 159 insertions(+), 189 deletions(-) diff --git a/crates/py_wheel_dist/BUILD.bazel b/crates/py_wheel_dist/BUILD.bazel index 18af91ed29..3691003761 100644 --- a/crates/py_wheel_dist/BUILD.bazel +++ b/crates/py_wheel_dist/BUILD.bazel @@ -10,6 +10,7 @@ rust_library( srcs = ["src/lib.rs"], edition = "2024", deps = [ + "@crates//:anyhow", "@crates//:argfile", "@crates//:lexopt", ], diff --git a/crates/py_wheel_dist/src/lib.rs b/crates/py_wheel_dist/src/lib.rs index ecf7e3ebf1..2351610030 100644 --- a/crates/py_wheel_dist/src/lib.rs +++ b/crates/py_wheel_dist/src/lib.rs @@ -1,57 +1,63 @@ //! Library for generating the output directory for `py_wheel_dist`. //! -//! Usage: -//! `py_wheel_dist --wheel --name_file --output ` -//! //! Reads the canonical wheel filename from `--name_file` (UTF-8, whitespace-trimmed), //! creates the `--output` directory, and copies `--wheel` to `/`. +// Errors must be returned, not panicked; see the `rust-lint` config in .bazelrc. +#![deny(clippy::expect_used, clippy::unwrap_used)] + use std::ffi::OsString; use std::fs; -use std::io; use std::path::{Path, PathBuf}; use std::process::ExitCode; -/// Validates and trims the wheel filename read from `--name_file`. -/// -/// Ensures the trimmed name is non-empty and a valid single file basename without -/// path separators (`/`, `\`) or parent directory traversal (`..`). -pub fn validate_wheel_name(raw_name: &str) -> io::Result<&str> { - let trimmed = raw_name.trim(); - if trimmed.is_empty() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "wheel name in --name_file is empty", - )); - } - if trimmed == "." - || trimmed == ".." - || trimmed.contains('/') - || trimmed.contains('\\') - || trimmed.contains("..") - || trimmed.contains('\0') - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("invalid wheel filename in --name_file: '{trimmed}'"), - )); - } - Ok(trimmed) +use anyhow::{Context, Result, bail}; +use lexopt::prelude::*; + +mod flags { + // keep-sorted start + pub const HELP: &str = "help"; + pub const HELP_SHORT: char = 'h'; + pub const NAME_FILE: &str = "name_file"; + pub const OUTPUT: &str = "output"; + pub const WHEEL: &str = "wheel"; + // keep-sorted end } -/// Creates the output directory and copies the wheel file to `/`. +/// Usage text for `--help`. `lexopt` does not generate help, so this is maintained by hand. +const USAGE: &str = "\ +Usage: py_wheel_dist --wheel --name_file --output + +Copy a wheel into a dist directory under its canonical filename. + +Options: + --wheel Path to the wheel to copy + --name_file File containing the sanitized wheel filename + --output Output directory to copy the wheel into + @ Read newline-delimited arguments from + -h, --help Print this help"; + +/// Creates the output directory and copies the wheel file to `/`, where +/// `wheel_name` is the whitespace-trimmed content of `name_file_path`. pub fn create_wheel_dist( wheel_path: &Path, name_file_path: &Path, output_dir: &Path, -) -> io::Result { - let raw_name = fs::read_to_string(name_file_path)?; - let wheel_name = validate_wheel_name(&raw_name)?; +) -> Result { + let raw_name = fs::read_to_string(name_file_path) + .with_context(|| format!("reading {}", name_file_path.display()))?; + let wheel_name = raw_name.trim(); - fs::create_dir_all(output_dir)?; + fs::create_dir_all(output_dir).with_context(|| format!("creating {}", output_dir.display()))?; let destination = output_dir.join(wheel_name); - fs::copy(wheel_path, &destination)?; + fs::copy(wheel_path, &destination).with_context(|| { + format!( + "copying {} to {}", + wheel_path.display(), + destination.display() + ) + })?; Ok(destination) } @@ -66,97 +72,60 @@ pub struct Options { pub output: PathBuf, } -fn parse_flagfile(content: &str, prefix: char) -> Vec { - content - .lines() - .filter(|line| !line.is_empty()) - .map(|line| argfile::Argument::parse_ref(line, prefix)) - .collect() -} - -fn parse_non_empty_path(parser: &mut lexopt::Parser, flag: &str) -> Result { - let val = parser.value().map_err(|e| e.to_string())?; - if val.is_empty() { - return Err(format!("empty value for {flag}")); - } - Ok(PathBuf::from(val)) +/// Result of parsing the command line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Cli { + /// `--help` was requested. + Help, + /// Copy the wheel with the given options. + Run(Options), } -/// Parses command-line arguments (excluding `argv[0]`) into [`Options`]. -/// -/// Supports `--flag value`, `--flag=value`, and `@flagfile` argument expansion. -pub fn parse_args(args: I) -> Result +/// Parses command-line arguments (excluding `argv[0]` and subcommand name) into [`Cli`]. +pub fn parse_args(args: I) -> Result where I: IntoIterator, T: Into, { - use lexopt::prelude::*; + let expanded = argfile::expand_args_from( + args.into_iter().map(Into::into), + argfile::parse_fromfile, + argfile::PREFIX, + ) + .context("reading @flagfile")?; - let raw_args: Vec = args.into_iter().map(Into::into).collect(); - if raw_args.iter().any(|arg| arg == "@") { - return Err("empty @flagfile path".to_string()); - } - let expanded = argfile::expand_args_from(raw_args.into_iter(), parse_flagfile, argfile::PREFIX) - .map_err(|e| format!("failed to read flagfile: {e}"))?; - - let mut wheel = None; - let mut name_file = None; - let mut output = None; + let mut wheel: Option = None; + let mut name_file: Option = None; + let mut output: Option = None; let mut parser = lexopt::Parser::from_args(expanded); - while let Some(arg) = parser.next().map_err(|e| e.to_string())? { + while let Some(arg) = parser.next()? { match arg { - Short('h') | Long("help") => { - return Err( - "Usage: py_wheel_dist --wheel --name_file --output " - .to_string(), - ); - } - Long("wheel") => wheel = Some(parse_non_empty_path(&mut parser, "--wheel")?), - Long("name_file") => { - name_file = Some(parse_non_empty_path(&mut parser, "--name_file")?); + Short(flags::HELP_SHORT) | Long(flags::HELP) => return Ok(Cli::Help), + Long(flags::WHEEL) => { + wheel = Some(PathBuf::from(parser.value()?)); } - Long("output") => output = Some(parse_non_empty_path(&mut parser, "--output")?), - Long(other) => { - let other = other.to_owned(); - if let Some(val) = parser.optional_value() { - return Err(format!( - "unexpected argument: --{other}={}", - val.to_string_lossy() - )); - } - return Err(format!("unexpected argument: --{other}")); + Long(flags::NAME_FILE) => { + name_file = Some(PathBuf::from(parser.value()?)); } - Short(ch) => { - if let Some(val) = parser.optional_value() { - return Err(format!( - "unexpected argument: -{ch}{}", - val.to_string_lossy() - )); - } - return Err(format!("unexpected argument: -{ch}")); - } - Value(val) => { - return Err(format!("unexpected argument: {}", val.to_string_lossy())); + Long(flags::OUTPUT) => { + output = Some(PathBuf::from(parser.value()?)); } + _ => bail!("{}", arg.unexpected()), } } - let wheel = wheel.ok_or_else(|| "missing required argument: --wheel".to_string())?; + let wheel = wheel.with_context(|| format!("missing required argument: --{}", flags::WHEEL))?; let name_file = - name_file.ok_or_else(|| "missing required argument: --name_file".to_string())?; - let output = output.ok_or_else(|| "missing required argument: --output".to_string())?; + name_file.with_context(|| format!("missing required argument: --{}", flags::NAME_FILE))?; + let output = + output.with_context(|| format!("missing required argument: --{}", flags::OUTPUT))?; - Ok(Options { + Ok(Cli::Run(Options { wheel, name_file, output, - }) -} - -/// Executes `py_wheel_dist` with the parsed [`Options`]. -pub fn run(options: &Options) -> io::Result { - create_wheel_dist(&options.wheel, &options.name_file, &options.output) + })) } /// Parses CLI arguments (excluding `argv[0]` and subcommand name), populates the wheel dist @@ -167,14 +136,18 @@ where T: Into, { let options = match parse_args(args) { - Ok(opts) => opts, - Err(msg) => { - eprintln!("py_wheel_dist: error: {msg}"); + Ok(Cli::Help) => { + println!("{USAGE}"); + return ExitCode::SUCCESS; + } + Ok(Cli::Run(opts)) => opts, + Err(err) => { + eprintln!("py_wheel_dist: error: {err:#}"); return ExitCode::FAILURE; } }; - if let Err(err) = run(&options) { - eprintln!("py_wheel_dist: error: {err}"); + if let Err(err) = create_wheel_dist(&options.wheel, &options.name_file, &options.output) { + eprintln!("py_wheel_dist: error: {err:#}"); return ExitCode::FAILURE; } ExitCode::SUCCESS diff --git a/tests/py_wheel_dist/py_wheel_dist_test.rs b/tests/py_wheel_dist/py_wheel_dist_test.rs index 9a3a6456e8..205c1b5d57 100644 --- a/tests/py_wheel_dist/py_wheel_dist_test.rs +++ b/tests/py_wheel_dist/py_wheel_dist_test.rs @@ -5,9 +5,7 @@ use std::path::{Path, PathBuf}; use std::process::ExitCode; use std::sync::atomic::{AtomicUsize, Ordering}; -use py_wheel_dist_lib::{ - Options, create_wheel_dist, parse_args, run, run_cli, validate_wheel_name, -}; +use py_wheel_dist_lib::{Cli, Options, create_wheel_dist, parse_args, run_cli}; static COUNTER: AtomicUsize = AtomicUsize::new(0); @@ -24,7 +22,7 @@ impl TempDir { std::process::id(), id )); - fs::create_dir_all(&path).unwrap(); + fs::create_dir_all(&path).expect("temp dir should be creatable"); Self { path } } @@ -39,17 +37,27 @@ impl Drop for TempDir { } } +fn parse_run(args: I) -> Options +where + I: IntoIterator, + T: Into, +{ + match parse_args(args).expect("arguments should parse") { + Cli::Run(opts) => opts, + Cli::Help => panic!("unexpected --help"), + } +} + #[test] fn test_parse_args_space_separated() { - let args = [ + let opts = parse_run([ "--wheel", "path/to/foo.whl", "--name_file", "path/to/name.txt", "--output", "path/to/dist", - ]; - let opts = parse_args(args).unwrap(); + ]); assert_eq!( opts, Options { @@ -62,12 +70,11 @@ fn test_parse_args_space_separated() { #[test] fn test_parse_args_equals_separated() { - let args = [ + let opts = parse_run([ "--wheel=path/to/foo.whl", "--name_file=path/to/name.txt", "--output=path/to/dist", - ]; - let opts = parse_args(args).unwrap(); + ]); assert_eq!( opts, Options { @@ -78,18 +85,26 @@ fn test_parse_args_equals_separated() { ); } +#[test] +fn test_parse_args_help() { + assert_eq!( + parse_args(["--help"]).expect("--help should parse"), + Cli::Help + ); + assert_eq!(parse_args(["-h"]).expect("-h should parse"), Cli::Help); +} + #[test] fn test_parse_args_flagfile() { let tmp = TempDir::new("flagfile"); let flagfile = tmp.path().join("args.params"); fs::write( &flagfile, - "--wheel\n\nmy_wheel.whl\n--name_file=my_name.txt\n\n--output\nout_dir\n", + "--wheel\nmy_wheel.whl\n--name_file=my_name.txt\n--output\nout_dir\n", ) - .unwrap(); + .expect("flagfile should be writable"); - let arg = format!("@{}", flagfile.display()); - let opts = parse_args([arg]).unwrap(); + let opts = parse_run([format!("@{}", flagfile.display())]); assert_eq!( opts, Options { @@ -104,16 +119,16 @@ fn test_parse_args_flagfile() { fn test_parse_args_mixed_flagfile_and_overrides() { let tmp = TempDir::new("mixed_flagfile"); let flagfile = tmp.path().join("args.params"); - fs::write(&flagfile, "--wheel\nfirst.whl\n--name_file\nname.txt\n").unwrap(); + fs::write(&flagfile, "--wheel\nfirst.whl\n--name_file\nname.txt\n") + .expect("flagfile should be writable"); let flagfile_arg = format!("@{}", flagfile.display()); - let opts = parse_args([ + let opts = parse_run([ flagfile_arg.as_str(), "--wheel=override.whl", "--output", "final_out", - ]) - .unwrap(); + ]); assert_eq!( opts, Options { @@ -126,14 +141,26 @@ fn test_parse_args_mixed_flagfile_and_overrides() { #[test] fn test_parse_args_missing_required_flags() { - let err_no_wheel = parse_args(["--name_file", "a", "--output", "b"]).unwrap_err(); - assert!(err_no_wheel.contains("--wheel"), "got: {err_no_wheel}"); + let err_no_wheel = + parse_args(["--name_file", "a", "--output", "b"]).expect_err("missing --wheel should fail"); + assert!( + err_no_wheel.to_string().contains("--wheel"), + "got: {err_no_wheel:#}" + ); - let err_no_name = parse_args(["--wheel", "a", "--output", "b"]).unwrap_err(); - assert!(err_no_name.contains("--name_file"), "got: {err_no_name}"); + let err_no_name = + parse_args(["--wheel", "a", "--output", "b"]).expect_err("missing --name_file should fail"); + assert!( + err_no_name.to_string().contains("--name_file"), + "got: {err_no_name:#}" + ); - let err_no_out = parse_args(["--wheel", "a", "--name_file", "b"]).unwrap_err(); - assert!(err_no_out.contains("--output"), "got: {err_no_out}"); + let err_no_out = + parse_args(["--wheel", "a", "--name_file", "b"]).expect_err("missing --output should fail"); + assert!( + err_no_out.to_string().contains("--output"), + "got: {err_no_out:#}" + ); } #[test] @@ -141,40 +168,12 @@ fn test_parse_args_missing_flag_values_and_unknown_args() { assert!(parse_args(["--wheel"]).is_err()); assert!(parse_args(["--name_file"]).is_err()); assert!(parse_args(["--output"]).is_err()); - assert!(parse_args(["--wheel="]).is_err()); - assert!(parse_args(["--name_file="]).is_err()); - assert!(parse_args(["--output="]).is_err()); assert!(parse_args(["--unknown", "val"]).is_err()); - let bogus_err = parse_args(["--bogus-flag=123"]).unwrap_err(); - assert!( - bogus_err.contains("unexpected argument: --bogus-flag=123"), - "got: {bogus_err}" - ); + assert!(parse_args(["-x"]).is_err()); assert!(parse_args(["positional_arg"]).is_err()); - assert!(parse_args(["@"]).is_err()); assert!(parse_args(["@nonexistent_flagfile_path_12345"]).is_err()); } -#[test] -fn test_validate_wheel_name() { - assert_eq!( - validate_wheel_name("example_pkg-0.0.1-py3-none-any.whl").unwrap(), - "example_pkg-0.0.1-py3-none-any.whl" - ); - assert_eq!( - validate_wheel_name(" \n\t example_pkg-1.2.3-py3-none-any.whl \r\n ").unwrap(), - "example_pkg-1.2.3-py3-none-any.whl" - ); - - assert!(validate_wheel_name("").is_err()); - assert!(validate_wheel_name(" \n\t ").is_err()); - assert!(validate_wheel_name(".").is_err()); - assert!(validate_wheel_name("..").is_err()); - assert!(validate_wheel_name("../evil.whl").is_err()); - assert!(validate_wheel_name("sub/wheel.whl").is_err()); - assert!(validate_wheel_name("sub\\wheel.whl").is_err()); -} - #[test] fn test_copy_wheel_into_new_and_existing_output_directory() { let tmp = TempDir::new("copy"); @@ -183,27 +182,32 @@ fn test_copy_wheel_into_new_and_existing_output_directory() { let out_dir = tmp.path().join("nested").join("dist_out"); let wheel_bytes = b"PK\x03\x04fake-wheel-archive-bytes"; - fs::write(&wheel_path, wheel_bytes).unwrap(); - fs::write(&name_file_path, " my_pkg-0.1.0-py3-none-any.whl\n").unwrap(); - - // 1. Copy into a new (non-existent) nested output directory. - let opts = Options { - wheel: wheel_path.clone(), - name_file: name_file_path.clone(), - output: out_dir.clone(), - }; - let copied_path = run(&opts).unwrap(); + fs::write(&wheel_path, wheel_bytes).expect("wheel should be writable"); + fs::write(&name_file_path, " my_pkg-0.1.0-py3-none-any.whl\n") + .expect("name file should be writable"); + + // 1. Copy into a new (non-existent) nested output directory; the name is trimmed. + let copied_path = create_wheel_dist(&wheel_path, &name_file_path, &out_dir) + .expect("copy into a new directory should succeed"); assert_eq!(copied_path, out_dir.join("my_pkg-0.1.0-py3-none-any.whl")); - assert_eq!(fs::read(&copied_path).unwrap(), wheel_bytes); + assert_eq!( + fs::read(&copied_path).expect("copied wheel should be readable"), + wheel_bytes + ); // 2. Copy into an already-existing output directory with updated wheel content. let updated_bytes = b"PK\x03\x04updated-wheel-archive-bytes"; - fs::write(&wheel_path, updated_bytes).unwrap(); - fs::write(&name_file_path, "my_pkg-0.2.0-py3-none-any.whl").unwrap(); + fs::write(&wheel_path, updated_bytes).expect("wheel should be writable"); + fs::write(&name_file_path, "my_pkg-0.2.0-py3-none-any.whl") + .expect("name file should be writable"); - let second_path = create_wheel_dist(&wheel_path, &name_file_path, &out_dir).unwrap(); + let second_path = create_wheel_dist(&wheel_path, &name_file_path, &out_dir) + .expect("copy into an existing directory should succeed"); assert_eq!(second_path, out_dir.join("my_pkg-0.2.0-py3-none-any.whl")); - assert_eq!(fs::read(&second_path).unwrap(), updated_bytes); + assert_eq!( + fs::read(&second_path).expect("copied wheel should be readable"), + updated_bytes + ); // 3. Verify run_cli returns ExitCode::SUCCESS and ExitCode::FAILURE. assert_eq!( @@ -217,11 +221,12 @@ fn test_copy_wheel_into_new_and_existing_output_directory() { ]), ExitCode::SUCCESS ); + assert_eq!(run_cli(["--help"]), ExitCode::SUCCESS); assert_eq!(run_cli(["--unknown"]), ExitCode::FAILURE); } #[test] -fn test_error_handling_missing_inputs_and_invalid_name() { +fn test_error_handling_missing_inputs() { let tmp = TempDir::new("errors"); let wheel_path = tmp.path().join("input.whl"); let name_file_path = tmp.path().join("name.txt"); @@ -231,15 +236,6 @@ fn test_error_handling_missing_inputs_and_invalid_name() { assert!(create_wheel_dist(&wheel_path, &name_file_path, &out_dir).is_err()); // Present name_file, missing wheel file. - fs::write(&name_file_path, "pkg-0.1.0-py3-none-any.whl").unwrap(); - assert!(create_wheel_dist(&wheel_path, &name_file_path, &out_dir).is_err()); - - // Present wheel file, empty/whitespace name_file. - fs::write(&wheel_path, b"wheel-data").unwrap(); - fs::write(&name_file_path, " \n ").unwrap(); - assert!(create_wheel_dist(&wheel_path, &name_file_path, &out_dir).is_err()); - - // Invalid traversal name in name_file. - fs::write(&name_file_path, "../escape.whl").unwrap(); + fs::write(&name_file_path, "pkg-0.1.0-py3-none-any.whl").expect("name file should be writable"); assert!(create_wheel_dist(&wheel_path, &name_file_path, &out_dir).is_err()); }