From 200bcfcf70d1b777d628b4e8a317424dfcb21b2d Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 15 Nov 2026 20:23:10 -0800 Subject: [PATCH] refactor(pypi): convert wheel_record_rewriter build tool to Rust Generating wheel `RECORD` metadata uses separate shell and PowerShell scripts across Unix and Windows hosts. Add `//crates/wheel_record_rewriter:wheel_record_rewriter_lib` using `csv`, register `wheel_record_rewriter` on `shrubbery`, and route `PyGenWheelRecord` through `get_shrubbery_tool` with the platform scripts as fallback. --- crates/shrubbery/BUILD.bazel | 1 + crates/shrubbery/src/lib.rs | 3 + crates/wheel_record_rewriter/BUILD.bazel | 21 + crates/wheel_record_rewriter/src/lib.rs | 350 +++++++++++++++ python/private/pypi/BUILD.bazel | 2 + python/private/pypi/gen_wheel_record.bzl | 46 +- tests/shrubbery/shrubbery_test.rs | 2 +- tests/wheel_record_rewriter/BUILD.bazel | 15 + .../wheel_record_rewriter_test.rs | 404 ++++++++++++++++++ 9 files changed, 827 insertions(+), 17 deletions(-) create mode 100644 crates/wheel_record_rewriter/BUILD.bazel create mode 100644 crates/wheel_record_rewriter/src/lib.rs create mode 100644 tests/wheel_record_rewriter/BUILD.bazel create mode 100644 tests/wheel_record_rewriter/wheel_record_rewriter_test.rs diff --git a/crates/shrubbery/BUILD.bazel b/crates/shrubbery/BUILD.bazel index aa3fb6d36a..e7c8eff447 100644 --- a/crates/shrubbery/BUILD.bazel +++ b/crates/shrubbery/BUILD.bazel @@ -15,6 +15,7 @@ rust_library( "//crates/py_console_script_gen:py_console_script_gen_lib", "//crates/py_wheel_dist:py_wheel_dist_lib", "//crates/venv_shebang_rewriter:venv_shebang_rewriter_lib", + "//crates/wheel_record_rewriter:wheel_record_rewriter_lib", "//crates/wheelmaker:wheelmaker_lib", "//crates/zip_main_maker:zip_main_maker_lib", "//crates/zipper:zipper_lib", diff --git a/crates/shrubbery/src/lib.rs b/crates/shrubbery/src/lib.rs index d29a7108e8..feb775a6a8 100644 --- a/crates/shrubbery/src/lib.rs +++ b/crates/shrubbery/src/lib.rs @@ -15,6 +15,7 @@ pub mod subcommands { pub const PY_CONSOLE_SCRIPT_GEN: &str = "py_console_script_gen"; pub const PY_WHEEL_DIST: &str = "py_wheel_dist"; pub const VENV_SHEBANG_REWRITER: &str = "venv_shebang_rewriter"; + pub const WHEEL_RECORD_REWRITER: &str = "wheel_record_rewriter"; pub const WHEELMAKER: &str = "wheelmaker"; pub const ZIP_MAIN_MAKER: &str = "zip_main_maker"; pub const ZIPPER: &str = "zipper"; @@ -29,6 +30,7 @@ pub const SUBCOMMANDS: &[&str] = &[ subcommands::PY_CONSOLE_SCRIPT_GEN, subcommands::PY_WHEEL_DIST, subcommands::VENV_SHEBANG_REWRITER, + subcommands::WHEEL_RECORD_REWRITER, subcommands::WHEELMAKER, subcommands::ZIP_MAIN_MAKER, subcommands::ZIPPER, @@ -78,6 +80,7 @@ where subcommands::PY_CONSOLE_SCRIPT_GEN => py_console_script_gen_lib::run_cli(iter), subcommands::PY_WHEEL_DIST => py_wheel_dist_lib::run_cli(iter), subcommands::VENV_SHEBANG_REWRITER => venv_shebang_rewriter_lib::run_cli(iter), + subcommands::WHEEL_RECORD_REWRITER => wheel_record_rewriter_lib::run_cli(iter), subcommands::WHEELMAKER => wheelmaker_lib::run_cli(iter), subcommands::ZIP_MAIN_MAKER => zip_main_maker_lib::run_cli(iter), subcommands::ZIPPER => zipper_lib::run_cli(iter), diff --git a/crates/wheel_record_rewriter/BUILD.bazel b/crates/wheel_record_rewriter/BUILD.bazel new file mode 100644 index 0000000000..f76e0f81cf --- /dev/null +++ b/crates/wheel_record_rewriter/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_rust//rust:defs.bzl", "rust_library") +load("//python/private:distribution_filegroup.bzl", "distribution_filegroup") # buildifier: disable=bzl-visibility + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_library( + name = "wheel_record_rewriter_lib", + srcs = ["src/lib.rs"], + edition = "2024", + deps = [ + "@crates//:anyhow", + "@crates//:csv", + "@crates//:lexopt", + ], +) + +distribution_filegroup( + name = "distribution", +) diff --git a/crates/wheel_record_rewriter/src/lib.rs b/crates/wheel_record_rewriter/src/lib.rs new file mode 100644 index 0000000000..b4d49f07a2 --- /dev/null +++ b/crates/wheel_record_rewriter/src/lib.rs @@ -0,0 +1,350 @@ +//! Rewrites a wheel's `RECORD` file to reflect its final installed layout. + +// Errors must be returned, not panicked; see the `rust-lint` config in .bazelrc. +#![deny(clippy::expect_used, clippy::unwrap_used)] + +use std::collections::HashSet; +use std::ffi::OsString; +use std::fs::File; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::str::FromStr; + +use anyhow::{Context, Result, anyhow, bail}; +use lexopt::prelude::*; + +mod flags { + // keep-sorted start + pub const HELP: &str = "help"; + pub const HELP_SHORT: char = 'h'; + // keep-sorted end +} + +/// Usage text for `--help`. `lexopt` does not generate help, so this is maintained by hand. +const USAGE: &str = "\ +Usage: wheel_record_rewriter [REWRITTEN_SCRIPT...] + +Rewrite a wheel RECORD file to reflect its final installed layout. + +Arguments: + Path to the original RECORD file + Path to write the rewritten RECORD file + Target layout: `unix` or `windows` + Basename of the wheel's `.data` directory + [REWRITTEN_SCRIPT...] Script names that had shebang rewriting applied (get a `.bat` suffix on Windows) + +Options: + -h, --help Print this help"; + +/// Target operating system layout for PEP 376 wheel `RECORD` path rewriting. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetOs { + Unix, + Windows, +} + +impl FromStr for TargetOs { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "unix" => Ok(Self::Unix), + "windows" => Ok(Self::Windows), + other => bail!("unsupported target_os '{other}'; expected 'unix' or 'windows'"), + } + } +} + +/// Replacement prefixes corresponding to PEP 376 installation paths. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ReplacementPrefixes { + data_repl: &'static str, + headers_repl: &'static str, + platlib_repl: &'static str, + purelib_repl: &'static str, + scripts_repl: &'static str, +} + +impl ReplacementPrefixes { + const fn for_target_os(target_os: TargetOs) -> Self { + match target_os { + TargetOs::Windows => Self { + data_repl: "../../", + headers_repl: "../../Include/", + platlib_repl: "", + purelib_repl: "", + scripts_repl: "../../Scripts/", + }, + TargetOs::Unix => Self { + data_repl: "../../../", + headers_repl: "../../../include/", + platlib_repl: "", + purelib_repl: "", + scripts_repl: "../../../bin/", + }, + } + } +} + +fn rewrite_entry_path_with_prefixes( + entry_path: &str, + target_os: TargetOs, + data_prefix: &str, + prefixes: &ReplacementPrefixes, + rewritten_scripts: &HashSet, +) -> String { + let Some(rest) = entry_path.strip_prefix(data_prefix) else { + return entry_path.to_owned(); + }; + + if let Some(after) = rest.strip_prefix("purelib/") { + format!("{}{after}", prefixes.purelib_repl) + } else if let Some(after) = rest.strip_prefix("platlib/") { + format!("{}{after}", prefixes.platlib_repl) + } else if let Some(spath) = rest.strip_prefix("scripts/") { + if target_os == TargetOs::Windows && rewritten_scripts.contains(spath) { + format!("{}{spath}.bat", prefixes.scripts_repl) + } else { + format!("{}{spath}", prefixes.scripts_repl) + } + } else if let Some(after) = rest.strip_prefix("headers/") { + format!("{}{after}", prefixes.headers_repl) + } else if let Some(after) = rest.strip_prefix("data/") { + format!("{}{after}", prefixes.data_repl) + } else { + entry_path.to_owned() + } +} + +/// Rewrites a single unescaped `RECORD` entry path (column 0). +pub fn rewrite_entry_path( + entry_path: &str, + target_os: TargetOs, + data_dir_basename: &str, + rewritten_scripts: &HashSet, +) -> String { + let data_prefix = format!("{data_dir_basename}/"); + let prefixes = ReplacementPrefixes::for_target_os(target_os); + rewrite_entry_path_with_prefixes( + entry_path, + target_os, + &data_prefix, + &prefixes, + rewritten_scripts, + ) +} + +/// Rewrites wheel `RECORD` CSV content from `reader` to `writer` with `\n` line endings. +/// +/// # Errors +/// +/// Returns an error if the input cannot be parsed as CSV or the output cannot be written. +pub fn rewrite_record_stream( + reader: R, + writer: W, + target_os: TargetOs, + data_dir_basename: &str, + rewritten_scripts: &HashSet, +) -> Result<()> { + let data_prefix = format!("{data_dir_basename}/"); + let prefixes = ReplacementPrefixes::for_target_os(target_os); + + let mut csv_reader = csv::ReaderBuilder::new() + .has_headers(false) + .flexible(true) + .from_reader(reader); + let mut csv_writer = csv::WriterBuilder::new() + .has_headers(false) + .flexible(true) + .terminator(csv::Terminator::Any(b'\n')) + .from_writer(writer); + + let mut record = csv::StringRecord::new(); + while csv_reader + .read_record(&mut record) + .context("reading RECORD entry")? + { + let rewritten_path = rewrite_entry_path_with_prefixes( + &record[0], + target_os, + &data_prefix, + &prefixes, + rewritten_scripts, + ); + csv_writer + .write_record(std::iter::once(rewritten_path.as_str()).chain(record.iter().skip(1))) + .with_context(|| format!("writing RECORD entry {rewritten_path}"))?; + } + + csv_writer.flush().context("flushing RECORD output") +} + +/// Rewrites an in-memory `RECORD` string and returns the rewritten UTF-8 string. +/// +/// # Errors +/// +/// Returns an error if the input cannot be parsed as CSV. +pub fn rewrite_record_str( + input: &str, + target_os: TargetOs, + data_dir_basename: &str, + rewritten_scripts: &HashSet, +) -> Result { + let mut output = Vec::with_capacity(input.len()); + rewrite_record_stream( + input.as_bytes(), + &mut output, + target_os, + data_dir_basename, + rewritten_scripts, + )?; + String::from_utf8(output).context("rewritten RECORD is not valid UTF-8") +} + +/// Rewrites the wheel `RECORD` file at `in_path` to `out_path`. +/// +/// # Errors +/// +/// Returns an error if `in_path` cannot be read, `out_path` cannot be created, or the +/// `RECORD` content cannot be parsed. +pub fn rewrite_record_files( + in_path: &Path, + out_path: &Path, + target_os: TargetOs, + data_dir_basename: &str, + rewritten_scripts: &HashSet, +) -> Result<()> { + let in_file = File::open(in_path).with_context(|| format!("opening {}", in_path.display()))?; + let out_file = + File::create(out_path).with_context(|| format!("creating {}", out_path.display()))?; + + rewrite_record_stream( + in_file, + out_file, + target_os, + data_dir_basename, + rewritten_scripts, + ) + .with_context(|| format!("rewriting {} to {}", in_path.display(), out_path.display())) +} + +/// Rewrites a wheel `RECORD` file using the provided [`Options`]. +/// +/// # Errors +/// +/// See [`rewrite_record_files`]. +pub fn rewrite_record(options: &Options) -> Result<()> { + rewrite_record_files( + &options.in_path, + &options.out_path, + options.target_os, + &options.data_dir_basename, + &options.rewritten_scripts, + ) +} + +/// Parsed command-line options for `wheel_record_rewriter`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Options { + /// Path to the original `RECORD` file. + pub in_path: PathBuf, + /// Path to write the rewritten `RECORD` file. + pub out_path: PathBuf, + /// Target installation layout. + pub target_os: TargetOs, + /// Basename of the wheel's `.data` directory (e.g. `foo-1.0.data`). + pub data_dir_basename: String, + /// Script names that had shebang rewriting applied. + pub rewritten_scripts: HashSet, +} + +/// Result of parsing the command line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Cli { + /// `--help` was requested. + Help, + /// Rewrite the `RECORD` file with the given options. + Run(Options), +} + +fn into_utf8(raw: OsString, what: &str) -> Result { + raw.into_string() + .map_err(|raw| anyhow!("invalid UTF-8 in {what}: {raw:?}")) +} + +/// Parses command-line arguments (excluding `argv[0]` and subcommand name) into [`Cli`]. +/// +/// Expected positional arguments: +/// ` [rewritten_scripts...]` +/// +/// # Errors +/// +/// Returns an error for unknown flags, fewer than four positional arguments, an unsupported +/// `target_os`, or non-UTF-8 `target_os`, `data_dir_basename`, or script names. +pub fn parse_args(args: I) -> Result +where + I: IntoIterator, + T: Into, +{ + let mut positional: Vec = Vec::new(); + + let mut parser = lexopt::Parser::from_args(args); + while let Some(arg) = parser.next()? { + match arg { + Short(flags::HELP_SHORT) | Long(flags::HELP) => return Ok(Cli::Help), + Value(val) => positional.push(val), + _ => bail!("{}", arg.unexpected()), + } + } + + let count = positional.len(); + let mut iter = positional.into_iter(); + let (Some(in_path), Some(out_path), Some(target_os), Some(data_dir_basename)) = + (iter.next(), iter.next(), iter.next(), iter.next()) + else { + bail!( + "expected at least 4 positional arguments ( ), got {count}" + ); + }; + let in_path = PathBuf::from(in_path); + let out_path = PathBuf::from(out_path); + let target_os = into_utf8(target_os, "target_os")?.parse::()?; + let data_dir_basename = into_utf8(data_dir_basename, "data_dir_basename")?; + let rewritten_scripts = iter + .map(|raw| into_utf8(raw, "rewritten script name")) + .collect::>>()?; + + Ok(Cli::Run(Options { + in_path, + out_path, + target_os, + data_dir_basename, + rewritten_scripts, + })) +} + +/// Parses CLI arguments (excluding `argv[0]` and subcommand name), rewrites the wheel `RECORD` +/// file, and returns the process exit code. +pub fn run_cli(args: I) -> ExitCode +where + I: IntoIterator, + T: Into, +{ + let options = match parse_args(args) { + Ok(Cli::Help) => { + println!("{USAGE}"); + return ExitCode::SUCCESS; + } + Ok(Cli::Run(opts)) => opts, + Err(err) => { + eprintln!("wheel_record_rewriter: error: {err:#}"); + return ExitCode::FAILURE; + } + }; + if let Err(err) = rewrite_record(&options) { + eprintln!("wheel_record_rewriter: error: {err:#}"); + return ExitCode::FAILURE; + } + ExitCode::SUCCESS +} diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index c55f35794a..3d1ba6c9da 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -505,6 +505,8 @@ bzl_library( deps = [ "//python/private:attributes", "//python/private:common", + "//python/private:toolchain_types", + "//python/private/toolchains:shrubbery_toolchain", ], ) diff --git a/python/private/pypi/gen_wheel_record.bzl b/python/private/pypi/gen_wheel_record.bzl index e85f6b98fa..366059b416 100644 --- a/python/private/pypi/gen_wheel_record.bzl +++ b/python/private/pypi/gen_wheel_record.bzl @@ -2,10 +2,17 @@ load("//python/private:attributes.bzl", "WINDOWS_CONSTRAINTS_PLAIN_ATTRS") load("//python/private:common.bzl", "is_windows_platform") +load("//python/private:toolchain_types.bzl", "SHRUBBERY_TOOLCHAIN_TYPE") +load("//python/private/toolchains:shrubbery_toolchain.bzl", "get_shrubbery_tool") def _gen_wheel_record_impl(ctx): is_windows = is_windows_platform(ctx) - rewriter_file = ctx.files._wheel_record_rewriter[0] + tool = get_shrubbery_tool( + ctx, + "wheel_record_rewriter", + ctx.attr._wheel_record_rewriter, + ) + rewriter_file = None if tool.is_rust else ctx.files._wheel_record_rewriter[0] out_files = [] for in_file in ctx.files.srcs: @@ -27,21 +34,25 @@ def _gen_wheel_record_impl(ctx): out_files.append(out_file) action_args = ctx.actions.args() - inputs = depset([in_file, rewriter_file]) - - if rewriter_file.path.endswith(".ps1"): - action_exe = "powershell.exe" - action_args.add_all([ - "-ExecutionPolicy", - "Bypass", - "-NoProfile", - "-File", - rewriter_file, - ]) + if tool.is_rust: + inputs = depset([in_file]) + action_exe = tool.files_to_run + action_toolchain = tool.toolchain + action_args.add_all(tool.prefix_args) else: - action_exe = ( - ctx.attr._wheel_record_rewriter[DefaultInfo].files_to_run - ) + inputs = depset([in_file, rewriter_file]) + action_toolchain = None + if rewriter_file.path.endswith(".ps1"): + action_exe = "powershell.exe" + action_args.add_all([ + "-ExecutionPolicy", + "Bypass", + "-NoProfile", + "-File", + rewriter_file, + ]) + else: + action_exe = tool.files_to_run action_args.add(in_file) action_args.add(out_file) @@ -56,7 +67,7 @@ def _gen_wheel_record_impl(ctx): arguments = [action_args], mnemonic = "PyRewriteWheelRecord", progress_message = "Rewriting wheel RECORD %{output}", - toolchain = None, + toolchain = action_toolchain, ) return [ @@ -81,4 +92,7 @@ gen_wheel_record = rule( cfg = "exec", ), }, + toolchains = [ + config_common.toolchain_type(SHRUBBERY_TOOLCHAIN_TYPE, mandatory = False), + ], ) diff --git a/tests/shrubbery/shrubbery_test.rs b/tests/shrubbery/shrubbery_test.rs index 0aab936c3c..199fbb463e 100644 --- a/tests/shrubbery/shrubbery_test.rs +++ b/tests/shrubbery/shrubbery_test.rs @@ -17,7 +17,7 @@ fn temp_test_dir(prefix: &str) -> PathBuf { #[test] fn test_subcommands_registry_sorted_and_complete() { - assert_eq!(SUBCOMMANDS.len(), 8); + assert_eq!(SUBCOMMANDS.len(), 9); let mut sorted = SUBCOMMANDS.to_vec(); sorted.sort_unstable(); assert_eq!(SUBCOMMANDS, sorted.as_slice()); diff --git a/tests/wheel_record_rewriter/BUILD.bazel b/tests/wheel_record_rewriter/BUILD.bazel new file mode 100644 index 0000000000..19d0fe4b68 --- /dev/null +++ b/tests/wheel_record_rewriter/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_rust//rust:defs.bzl", "rust_test") + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_test( + name = "wheel_record_rewriter_test", + size = "small", + srcs = ["wheel_record_rewriter_test.rs"], + edition = "2024", + deps = [ + "//crates/wheel_record_rewriter:wheel_record_rewriter_lib", + ], +) diff --git a/tests/wheel_record_rewriter/wheel_record_rewriter_test.rs b/tests/wheel_record_rewriter/wheel_record_rewriter_test.rs new file mode 100644 index 0000000000..20761e1544 --- /dev/null +++ b/tests/wheel_record_rewriter/wheel_record_rewriter_test.rs @@ -0,0 +1,404 @@ +use std::collections::HashSet; +use std::env; +use std::ffi::OsString; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use wheel_record_rewriter_lib::{ + Cli, Options, TargetOs, parse_args, rewrite_entry_path, rewrite_record, rewrite_record_files, + rewrite_record_str, run_cli, +}; + +static TEST_COUNTER: AtomicUsize = AtomicUsize::new(0); + +/// Parses `args` and unwraps the `Cli::Run` variant. +fn parse_run(args: I) -> Options +where + I: IntoIterator, + T: Into, +{ + match parse_args(args).expect("arguments should parse successfully") { + Cli::Run(opts) => opts, + Cli::Help => panic!("expected Cli::Run, got Cli::Help"), + } +} + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn new(prefix: &str) -> Self { + let id = TEST_COUNTER.fetch_add(1, Ordering::SeqCst); + let path = env::temp_dir().join(format!( + "wheel_record_rewriter_test_{prefix}_{}_{id}", + std::process::id(), + )); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).unwrap(); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +const SAMPLE_INPUT_RECORD: &str = "\ +foo-1.0.data/purelib/pkg/__init__.py,sha256=abc,100 +foo-1.0.data/purelib/pkg/module.py,sha256=def,200 +foo-1.0.data/platlib/pkg/_ext.so,sha256=ghi,300 +foo-1.0.data/data/pkg/data.txt,sha256=111,10 +foo-1.0.data/headers/pkg/header.h,sha256=222,20 +foo-1.0.data/scripts/foo,sha256=444,40 +foo-1.0.data/scripts/my_script.sh,sha256=333,30 +foo-1.0.data/scripts/plain_tool,sha256=555,50 +foo-1.0.data/scripts/my_py_script.py,sha256=666,60 +\"foo-1.0.data/purelib/pkg/my file.py\",sha256=abc,100 +\"foo-1.0.data/scripts/my tool\",sha256=def,200 +\"foo-1.0.data/headers/my,header.h\",sha256=ghi,300 +\"foo-1.0.data/scripts/quote\"\"tool\",sha256=jkl,400 +foo-1.0.data/custom_dir/custom.txt,sha256=xyz,123 +top_level/__init__.py,sha256=aaa,50 +foo-1.0.dist-info/METADATA,sha256=bbb,60 +foo-1.0.dist-info/RECORD,, +"; + +const EXPECTED_UNIX_RECORD: &str = "\ +pkg/__init__.py,sha256=abc,100 +pkg/module.py,sha256=def,200 +pkg/_ext.so,sha256=ghi,300 +../../../pkg/data.txt,sha256=111,10 +../../../include/pkg/header.h,sha256=222,20 +../../../bin/foo,sha256=444,40 +../../../bin/my_script.sh,sha256=333,30 +../../../bin/plain_tool,sha256=555,50 +../../../bin/my_py_script.py,sha256=666,60 +pkg/my file.py,sha256=abc,100 +../../../bin/my tool,sha256=def,200 +\"../../../include/my,header.h\",sha256=ghi,300 +\"../../../bin/quote\"\"tool\",sha256=jkl,400 +foo-1.0.data/custom_dir/custom.txt,sha256=xyz,123 +top_level/__init__.py,sha256=aaa,50 +foo-1.0.dist-info/METADATA,sha256=bbb,60 +foo-1.0.dist-info/RECORD,, +"; + +const EXPECTED_WINDOWS_WITH_REWRITTEN_SCRIPTS_RECORD: &str = "\ +pkg/__init__.py,sha256=abc,100 +pkg/module.py,sha256=def,200 +pkg/_ext.so,sha256=ghi,300 +../../pkg/data.txt,sha256=111,10 +../../Include/pkg/header.h,sha256=222,20 +../../Scripts/foo.bat,sha256=444,40 +../../Scripts/my_script.sh,sha256=333,30 +../../Scripts/plain_tool,sha256=555,50 +../../Scripts/my_py_script.py.bat,sha256=666,60 +pkg/my file.py,sha256=abc,100 +../../Scripts/my tool.bat,sha256=def,200 +\"../../Include/my,header.h\",sha256=ghi,300 +\"../../Scripts/quote\"\"tool.bat\",sha256=jkl,400 +foo-1.0.data/custom_dir/custom.txt,sha256=xyz,123 +top_level/__init__.py,sha256=aaa,50 +foo-1.0.dist-info/METADATA,sha256=bbb,60 +foo-1.0.dist-info/RECORD,, +"; + +const EXPECTED_WINDOWS_NO_REWRITTEN_SCRIPTS_RECORD: &str = "\ +pkg/__init__.py,sha256=abc,100 +pkg/module.py,sha256=def,200 +pkg/_ext.so,sha256=ghi,300 +../../pkg/data.txt,sha256=111,10 +../../Include/pkg/header.h,sha256=222,20 +../../Scripts/foo,sha256=444,40 +../../Scripts/my_script.sh,sha256=333,30 +../../Scripts/plain_tool,sha256=555,50 +../../Scripts/my_py_script.py,sha256=666,60 +pkg/my file.py,sha256=abc,100 +../../Scripts/my tool,sha256=def,200 +\"../../Include/my,header.h\",sha256=ghi,300 +\"../../Scripts/quote\"\"tool\",sha256=jkl,400 +foo-1.0.data/custom_dir/custom.txt,sha256=xyz,123 +top_level/__init__.py,sha256=aaa,50 +foo-1.0.dist-info/METADATA,sha256=bbb,60 +foo-1.0.dist-info/RECORD,, +"; + +#[test] +fn test_unix_rewrite_full_record() { + let temp = TempDir::new("unix"); + let in_path = temp.path().join("input_RECORD"); + let out_path = temp.path().join("unix_RECORD"); + fs::write(&in_path, SAMPLE_INPUT_RECORD).unwrap(); + + let args: Vec = vec![ + in_path.as_os_str().to_owned(), + out_path.as_os_str().to_owned(), + OsString::from("unix"), + OsString::from("foo-1.0.data"), + OsString::from("foo"), + OsString::from("my tool"), + OsString::from("my_py_script.py"), + OsString::from("quote\"tool"), + ]; + let opts = parse_run(&args); + assert_eq!(opts.target_os, TargetOs::Unix); + rewrite_record(&opts).unwrap(); + + let actual = fs::read_to_string(&out_path).unwrap(); + assert_eq!(actual, EXPECTED_UNIX_RECORD); +} + +#[test] +fn test_windows_rewrite_with_rewritten_scripts() { + let temp = TempDir::new("win_scripts"); + let in_path = temp.path().join("input_RECORD"); + let out_path = temp.path().join("win_RECORD"); + fs::write(&in_path, SAMPLE_INPUT_RECORD).unwrap(); + + let args: Vec = vec![ + in_path.as_os_str().to_owned(), + out_path.as_os_str().to_owned(), + OsString::from("windows"), + OsString::from("foo-1.0.data"), + OsString::from("foo"), + OsString::from("my tool"), + OsString::from("my_py_script.py"), + OsString::from("quote\"tool"), + ]; + let opts = parse_run(&args); + assert_eq!(opts.target_os, TargetOs::Windows); + rewrite_record(&opts).unwrap(); + + let actual = fs::read_to_string(&out_path).unwrap(); + assert_eq!(actual, EXPECTED_WINDOWS_WITH_REWRITTEN_SCRIPTS_RECORD); +} + +#[test] +fn test_windows_rewrite_without_rewritten_scripts() { + let temp = TempDir::new("win_no_scripts"); + let in_path = temp.path().join("input_RECORD"); + let out_path = temp.path().join("win_no_rewrite_RECORD"); + fs::write(&in_path, SAMPLE_INPUT_RECORD).unwrap(); + + let args: Vec = vec![ + in_path.as_os_str().to_owned(), + out_path.as_os_str().to_owned(), + OsString::from("windows"), + OsString::from("foo-1.0.data"), + ]; + let opts = parse_run(&args); + rewrite_record(&opts).unwrap(); + + let actual = fs::read_to_string(&out_path).unwrap(); + assert_eq!(actual, EXPECTED_WINDOWS_NO_REWRITTEN_SCRIPTS_RECORD); +} + +#[test] +fn test_crlf_input_normalized_to_lf_output() { + let temp = TempDir::new("crlf"); + let in_path = temp.path().join("crlf_RECORD"); + let out_path = temp.path().join("out_RECORD"); + + let crlf_input = SAMPLE_INPUT_RECORD.replace('\n', "\r\n"); + fs::write(&in_path, crlf_input).unwrap(); + + let rewritten_scripts: HashSet = ["foo", "my tool", "my_py_script.py", "quote\"tool"] + .into_iter() + .map(String::from) + .collect(); + rewrite_record_files( + &in_path, + &out_path, + TargetOs::Windows, + "foo-1.0.data", + &rewritten_scripts, + ) + .unwrap(); + + let actual_bytes = fs::read(&out_path).unwrap(); + assert!( + !actual_bytes.contains(&b'\r'), + "output RECORD should only contain LF line endings" + ); + assert_eq!( + String::from_utf8(actual_bytes).unwrap(), + EXPECTED_WINDOWS_WITH_REWRITTEN_SCRIPTS_RECORD + ); +} + +#[test] +fn test_custom_data_dir_and_csv_quoting() { + let input = "\ +\"bar-2.0.data/platlib/pkg/my,ext.pyd\",sha256=123,456 +\"bar-2.0.data/custom_dir/quoted,custom.txt\",sha256=789,101 +bar-2.0.data/custom_dir/unquoted_custom.txt,sha256=abc,202 +other-1.0.data/purelib/pkg/other.py,sha256=def,303 +"; + let expected = "\ +\"pkg/my,ext.pyd\",sha256=123,456 +\"bar-2.0.data/custom_dir/quoted,custom.txt\",sha256=789,101 +bar-2.0.data/custom_dir/unquoted_custom.txt,sha256=abc,202 +other-1.0.data/purelib/pkg/other.py,sha256=def,303 +"; + let rewritten = + rewrite_record_str(input, TargetOs::Unix, "bar-2.0.data", &HashSet::new()).unwrap(); + assert_eq!(rewritten, expected); +} + +#[test] +fn test_parse_args_errors_on_missing_or_invalid_args() { + assert!(parse_args([""; 0]).is_err()); + assert!(parse_args(["in"]).is_err()); + assert!(parse_args(["in", "out"]).is_err()); + assert!(parse_args(["in", "out", "unix"]).is_err()); + assert!(parse_args(["--unknown-flag", "in", "out", "unix", "pkg.data"]).is_err()); + assert!(parse_args(["-x", "in", "out", "unix", "pkg.data"]).is_err()); + + let err = parse_args(["in", "out", "macos", "pkg.data"]).unwrap_err(); + let msg = format!("{err:#}"); + assert!(msg.contains("unsupported target_os 'macos'"), "err: {msg}"); +} + +#[test] +fn test_parse_args_help() { + assert_eq!(parse_args(["--help"]).unwrap(), Cli::Help); + assert_eq!(parse_args(["-h"]).unwrap(), Cli::Help); + assert_eq!( + parse_args(["in", "out", "unix", "pkg.data", "--help"]).unwrap(), + Cli::Help + ); + assert_eq!(run_cli(["--help"]), ExitCode::SUCCESS); +} + +#[test] +fn test_parse_args_deduplicates_rewritten_scripts_and_supports_double_dash() { + let args = [ + "--", + "-in_file", + "-out_file", + "windows", + "pkg-1.0.data", + "tool_a", + "tool_b", + "tool_a", + ]; + let opts = parse_run(args); + assert_eq!(opts.in_path, PathBuf::from("-in_file")); + assert_eq!(opts.out_path, PathBuf::from("-out_file")); + assert_eq!(opts.target_os, TargetOs::Windows); + assert_eq!(opts.data_dir_basename, "pkg-1.0.data"); + assert_eq!(opts.rewritten_scripts.len(), 2); + assert!(opts.rewritten_scripts.contains("tool_a")); + assert!(opts.rewritten_scripts.contains("tool_b")); +} + +#[test] +fn test_missing_input_file_returns_error() { + let temp = TempDir::new("missing_in"); + let in_path = temp.path().join("does_not_exist"); + let out_path = temp.path().join("out_RECORD"); + + let res = rewrite_record_files( + &in_path, + &out_path, + TargetOs::Unix, + "foo-1.0.data", + &HashSet::new(), + ); + assert!(res.is_err()); +} + +#[test] +fn test_empty_input_blank_lines_and_missing_trailing_newline() { + let empty_out = + rewrite_record_str("\n\r\n", TargetOs::Unix, "foo-1.0.data", &HashSet::new()).unwrap(); + assert_eq!(empty_out, ""); + + let no_newline_in = "foo-1.0.data/purelib/mod.py,sha256=abc,12"; + let no_newline_out = rewrite_record_str( + no_newline_in, + TargetOs::Unix, + "foo-1.0.data", + &HashSet::new(), + ) + .unwrap(); + assert_eq!(no_newline_out, "mod.py,sha256=abc,12\n"); +} + +#[test] +fn test_data_dir_basename_named_data_and_prefix_boundary() { + let scripts: HashSet = ["cli"].into_iter().map(String::from).collect(); + let input = "\ +data/data/share/doc.txt,sha256=111,10 +data/scripts/cli,sha256=222,20 +data_extra/purelib/untouched.py,sha256=333,30 +\"data_extra/purelib/quoted,untouched.py\",sha256=444,40 +"; + let expected = "\ +../../share/doc.txt,sha256=111,10 +../../Scripts/cli.bat,sha256=222,20 +data_extra/purelib/untouched.py,sha256=333,30 +\"data_extra/purelib/quoted,untouched.py\",sha256=444,40 +"; + let actual = rewrite_record_str(input, TargetOs::Windows, "data", &scripts).unwrap(); + assert_eq!(actual, expected); +} + +#[test] +fn test_utf8_filenames_and_rewrite_entry_path() { + let scripts: HashSet = ["outil_é"].into_iter().map(String::from).collect(); + assert_eq!( + rewrite_entry_path( + "foo-1.0.data/scripts/outil_é", + TargetOs::Windows, + "foo-1.0.data", + &scripts + ), + "../../Scripts/outil_é.bat" + ); + + let input = "\ +foo-1.0.data/purelib/pkg/café.py,sha256=abc,10 +\"foo-1.0.data/scripts/outil_é\",sha256=def,20 +"; + let expected_win = "\ +pkg/café.py,sha256=abc,10 +../../Scripts/outil_é.bat,sha256=def,20 +"; + let actual_win = + rewrite_record_str(input, TargetOs::Windows, "foo-1.0.data", &scripts).unwrap(); + assert_eq!(actual_win, expected_win); +} + +#[test] +fn test_run_cli_exit_codes() { + let temp = TempDir::new("run_cli"); + let in_path = temp.path().join("in_RECORD"); + let out_path = temp.path().join("out_RECORD"); + fs::write(&in_path, "foo-1.0.data/scripts/app,sha256=abc,10\n").unwrap(); + + let ok_code = run_cli([ + in_path.as_os_str(), + out_path.as_os_str(), + OsString::from("windows").as_os_str(), + OsString::from("foo-1.0.data").as_os_str(), + OsString::from("app").as_os_str(), + ]); + assert_eq!(ok_code, ExitCode::SUCCESS); + assert_eq!( + fs::read_to_string(&out_path).unwrap(), + "../../Scripts/app.bat,sha256=abc,10\n" + ); + + let err_code = run_cli(["missing_args"]); + assert_eq!(err_code, ExitCode::FAILURE); +}