Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions crates/extract_wheel_files/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
load("@rules_rust//rust:defs.bzl", "rust_library")
load("//python/private:distribution_filegroup.bzl", "distribution_filegroup") # buildifier: disable=bzl-visibility

package(default_visibility = ["//:__subpackages__"])

licenses(["notice"])

rust_library(
name = "extract_wheel_files_lib",
srcs = ["src/lib.rs"],
edition = "2024",
deps = [
"@crates//:anyhow",
"@crates//:csv",
"@crates//:lexopt",
"@crates//:regex",
"@crates//:zip",
],
)

distribution_filegroup(
name = "distribution",
)
252 changes: 252 additions & 0 deletions crates/extract_wheel_files/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,252 @@
//! Library for extracting files from a Python wheel (`.whl`) archive matching a regex pattern.
//!
//! Reads the `.dist-info/RECORD` file from `<wheel>`, filters column 0 file paths using
//! start-anchored regex matching (`re.match` semantics), and extracts matching files into
//! `<out_dir>`.

// Errors must be returned, not panicked; see the `rust-lint` config in .bazelrc.
#![deny(clippy::expect_used, clippy::unwrap_used)]

use std::ffi::OsString;
use std::fs::{self, File};
use std::io::{self, BufReader, Read};
use std::path::{Path, PathBuf};
use std::process::ExitCode;

use anyhow::{Context, Result, bail};
use lexopt::prelude::*;
use regex::Regex;
use zip::ZipArchive;

mod flags {
// keep-sorted start
pub const HELP: &str = "help";
pub const HELP_SHORT: char = 'h';
// keep-sorted end
}

/// Usage text for `--help`. `lexopt` does not generate help, so this is maintained by hand.
const USAGE: &str = "\
Usage: extract_wheel_files <wheel> <out_dir> [regex_pattern]

Extract the files listed in a wheel's RECORD that match a regex pattern.

Arguments:
<wheel> Path to the wheel to extract from
<out_dir> Directory to extract the matching files into
[regex_pattern] Start-anchored regex the RECORD paths must match (default: match all)

Options:
-h, --help Print this help";

/// Compiles a start-anchored regular expression (`Python re.match` semantics).
pub fn compile_pattern(regex_pattern: &str) -> Result<Regex> {
let anchored_pattern = format!(r"\A(?:{regex_pattern})");
Regex::new(&anchored_pattern).with_context(|| format!("invalid regex '{regex_pattern}'"))
}

/// Parses CSV `RECORD` bytes and returns the list of file paths (column 0).
pub fn parse_record(record_bytes: &[u8]) -> Result<Vec<String>> {
let mut reader = csv::ReaderBuilder::new()
.has_headers(false)
.flexible(true)
.from_reader(record_bytes);
let mut paths = Vec::new();
for result in reader.records() {
let record = result.context("parsing RECORD CSV")?;
if let Some(first) = record.get(0) {
paths.push(first.to_owned());
}
}
Ok(paths)
}

/// Reads the `.dist-info/RECORD` file from `whl_path` and returns the list of file paths (column 0).
pub fn get_record(whl_path: &Path) -> Result<Vec<String>> {
let mut archive = open_wheel_archive(whl_path)?;
// zip 9 returns `Result<Cow<str>>` per entry name; a bad name means the
// archive is malformed, so surface it rather than skipping the entry.
let mut record_names: Vec<String> = Vec::new();
for name in archive.file_names() {
let name = name.with_context(|| format!("{}: invalid entry name", whl_path.display()))?;
if name.ends_with(".dist-info/RECORD") {
record_names.push(name.into_owned());
}
}

let [record_name] = record_names.as_slice() else {
bail!(
"{} doesn't contain exactly one .dist-info/RECORD",
whl_path.display()
);
};

let mut record_bytes = Vec::new();
archive
.by_name(record_name)
.and_then(|mut entry| Ok(entry.read_to_end(&mut record_bytes)?))
.with_context(|| format!("reading {record_name} from {}", whl_path.display()))?;

parse_record(&record_bytes)
}

/// Filters `whl_record` entries by a pre-compiled `regex`.
pub fn get_files<I, S>(whl_record: I, regex: &Regex) -> Vec<String>
where
I: IntoIterator<Item = S>,
S: AsRef<str>,
{
whl_record
.into_iter()
.filter(|entry| regex.is_match(entry.as_ref()))
.map(|entry| entry.as_ref().to_owned())
.collect()
}

/// Extracts `files` from `whl_path` into `outdir`.
pub fn extract_files<I, S>(whl_path: &Path, files: I, outdir: &Path) -> Result<()>
where
I: IntoIterator<Item = S>,
S: AsRef<str>,
{
fs::create_dir_all(outdir).with_context(|| format!("creating {}", outdir.display()))?;

let mut archive = open_wheel_archive(whl_path)?;
for file in files {
let entry_name = file.as_ref();
let mut entry = archive
.by_name(entry_name)
.with_context(|| format!("opening '{entry_name}' in {}", whl_path.display()))?;

let dest_path = sanitize_extract_path(outdir, entry_name);
if entry.is_dir() {
fs::create_dir_all(&dest_path)
.with_context(|| format!("creating {}", dest_path.display()))?;
continue;
}

if let Some(parent) = dest_path.parent() {
fs::create_dir_all(parent).with_context(|| format!("creating {}", parent.display()))?;
}

let mut out_file = File::create(&dest_path)
.with_context(|| format!("creating {}", dest_path.display()))?;
io::copy(&mut entry, &mut out_file)
.with_context(|| format!("extracting '{entry_name}' to {}", dest_path.display(),))?;
}

Ok(())
}

fn open_wheel_archive(whl_path: &Path) -> Result<ZipArchive<BufReader<File>>> {
let file = File::open(whl_path).with_context(|| format!("opening {}", whl_path.display()))?;
ZipArchive::new(BufReader::new(file))
.with_context(|| format!("{} is not a valid zip file", whl_path.display()))
}

/// Mirrors `zipfile.ZipFile._extract_member`: drops empty, `.`, and `..` path components so an
/// entry cannot escape `outdir`.
fn sanitize_extract_path(outdir: &Path, entry_name: &str) -> PathBuf {
let mut dest = outdir.to_path_buf();
dest.extend(
entry_name
.split(['/', '\\'])
.filter(|part| !part.is_empty() && *part != "." && *part != ".."),
);
dest
}

/// Parsed CLI options for `extract_wheel_files`.
#[derive(Debug, Clone)]
pub struct Options {
/// Path to the input `.whl` file.
pub whl_path: PathBuf,
/// Directory to extract matching files into.
pub outdir: PathBuf,
/// Start-anchored pattern RECORD paths must match.
pub regex: Regex,
}

/// Result of parsing the command line.
#[derive(Debug, Clone)]
pub enum Cli {
/// `--help` was requested.
Help,
/// Extract files with the given options.
Run(Options),
}

/// Parses command-line arguments (excluding `argv[0]` and subcommand name) into [`Cli`].
pub fn parse_args<I, T>(args: I) -> Result<Cli>
where
I: IntoIterator<Item = T>,
T: Into<OsString>,
{
let mut positional: Vec<OsString> = Vec::new();

let mut parser = lexopt::Parser::from_args(args.into_iter().map(Into::into));
while let Some(arg) = parser.next()? {
match arg {
Short(flags::HELP_SHORT) | Long(flags::HELP) => return Ok(Cli::Help),
Value(val) => positional.push(val),
_ => bail!("{}", arg.unexpected()),
}
}

if positional.len() != 2 && positional.len() != 3 {
bail!(
"expected 2 or 3 positional arguments (<wheel> <out_dir> [regex_pattern]), got {}",
positional.len()
);
}

// The length check above guarantees the first two positionals exist.
let whl_path = PathBuf::from(positional.remove(0));
let outdir = PathBuf::from(positional.remove(0));
let regex_pattern = match positional.pop() {
Some(raw) => raw
.into_string()
.ok()
.context("regex pattern is not valid UTF-8")?,
None => String::new(),
};
let regex = compile_pattern(&regex_pattern)?;

Ok(Cli::Run(Options {
whl_path,
outdir,
regex,
}))
}

/// Executes the full `extract_wheel_files` workflow for the given options.
pub fn run(options: &Options) -> Result<()> {
let record = get_record(&options.whl_path)?;
let files = get_files(&record, &options.regex);
extract_files(&options.whl_path, &files, &options.outdir)
}

/// Parses CLI arguments (excluding `argv[0]` and subcommand name), extracts matching wheel files,
/// and returns the process exit code.
pub fn run_cli<I, T>(args: I) -> ExitCode
where
I: IntoIterator<Item = T>,
T: Into<OsString>,
{
let options = match parse_args(args) {
Ok(Cli::Help) => {
println!("{USAGE}");
return ExitCode::SUCCESS;
}
Ok(Cli::Run(opts)) => opts,
Err(err) => {
eprintln!("extract_wheel_files: error: {err:#}");
return ExitCode::FAILURE;
}
};
if let Err(err) = run(&options) {
eprintln!("extract_wheel_files: error: {err:#}");
return ExitCode::FAILURE;
}
ExitCode::SUCCESS
}
1 change: 1 addition & 0 deletions crates/shrubbery/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ rust_library(
edition = "2024",
deps = [
"//crates/exe_zip_maker:exe_zip_maker_lib",
"//crates/extract_wheel_files:extract_wheel_files_lib",
"//crates/py_console_script_gen:py_console_script_gen_lib",
"//crates/py_wheel_dist:py_wheel_dist_lib",
"//crates/wheelmaker:wheelmaker_lib",
Expand Down
3 changes: 3 additions & 0 deletions crates/shrubbery/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use std::process::ExitCode;
pub mod subcommands {
// keep-sorted start case=no
pub const EXE_ZIP_MAKER: &str = "exe_zip_maker";
pub const EXTRACT_WHEEL_FILES: &str = "extract_wheel_files";
pub const PY_CONSOLE_SCRIPT_GEN: &str = "py_console_script_gen";
pub const PY_WHEEL_DIST: &str = "py_wheel_dist";
pub const WHEELMAKER: &str = "wheelmaker";
Expand All @@ -23,6 +24,7 @@ pub mod subcommands {
pub const SUBCOMMANDS: &[&str] = &[
// keep-sorted start case=no
subcommands::EXE_ZIP_MAKER,
subcommands::EXTRACT_WHEEL_FILES,
subcommands::PY_CONSOLE_SCRIPT_GEN,
subcommands::PY_WHEEL_DIST,
subcommands::WHEELMAKER,
Expand Down Expand Up @@ -70,6 +72,7 @@ where
match sub_raw.as_ref() {
// keep-sorted start case=no
subcommands::EXE_ZIP_MAKER => exe_zip_maker_lib::run_cli(iter),
subcommands::EXTRACT_WHEEL_FILES => extract_wheel_files_lib::run_cli(iter),
subcommands::PY_CONSOLE_SCRIPT_GEN => py_console_script_gen_lib::run_cli(iter),
subcommands::PY_WHEEL_DIST => py_wheel_dist_lib::run_cli(iter),
subcommands::WHEELMAKER => wheelmaker_lib::run_cli(iter),
Expand Down
4 changes: 4 additions & 0 deletions python/private/whl_filegroup/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,8 @@ py_binary(
bzl_library(
name = "whl_filegroup",
srcs = ["whl_filegroup.bzl"],
deps = [
"//python/private:toolchain_types",
"//python/private/toolchains:shrubbery_toolchain",
],
)
16 changes: 14 additions & 2 deletions python/private/whl_filegroup/whl_filegroup.bzl
Original file line number Diff line number Diff line change
@@ -1,16 +1,25 @@
"""Implementation of whl_filegroup rule."""

load("//python/private:toolchain_types.bzl", "SHRUBBERY_TOOLCHAIN_TYPE")
load("//python/private/toolchains:shrubbery_toolchain.bzl", "get_shrubbery_tool")

def _whl_filegroup_impl(ctx):
out_dir = ctx.actions.declare_directory(ctx.attr.name)
tool = get_shrubbery_tool(
ctx,
"extract_wheel_files",
ctx.attr._extract_wheel_files_tool,
)
ctx.actions.run(
outputs = [out_dir],
inputs = [ctx.file.whl],
arguments = [
arguments = tool.prefix_args + [
ctx.file.whl.path,
out_dir.path,
ctx.attr.pattern,
],
executable = ctx.executable._extract_wheel_files_tool,
executable = tool.files_to_run,
toolchain = tool.toolchain,
mnemonic = "PyExtractWheelFiles",
progress_message = "Extracting %s files from %s" % (ctx.attr.pattern, ctx.file.whl.short_path),
)
Expand Down Expand Up @@ -61,4 +70,7 @@ from the already extracted whl file.
executable = True,
),
},
toolchains = [
config_common.toolchain_type(SHRUBBERY_TOOLCHAIN_TYPE, mandatory = False),
],
)
20 changes: 20 additions & 0 deletions tests/extract_wheel_files/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
load("@rules_rust//rust:defs.bzl", "rust_test")

package(default_visibility = ["//:__subpackages__"])

licenses(["notice"])

rust_test(
name = "extract_wheel_files_test",
size = "small",
srcs = ["extract_wheel_files_test.rs"],
data = [
"//crates/shrubbery",
"//examples/wheel:minimal_with_py_package",
],
edition = "2024",
deps = [
"//crates/extract_wheel_files:extract_wheel_files_lib",
"@crates//:zip",
],
)
Loading
Loading