diff --git a/crates/core/src/format.rs b/crates/core/src/format.rs index 56ee8403..26f6c85b 100644 --- a/crates/core/src/format.rs +++ b/crates/core/src/format.rs @@ -143,6 +143,14 @@ pub trait Format: Send + Sync { None } + /// The fields of [`Self::manifest_summary`] of which a hosted repository + /// holds a bounded number of distinct values + /// ([`forge_plugin_api::FormatPlugin::summary_caps`]): a publish or an + /// import that would add a value past one is refused. None by default. + fn summary_caps(&self) -> &'static [forge_plugin_api::SummaryCap] { + &[] + } + /// Whether `id` is a well-formed package id of this format. Read routes /// answer 404 for anything else, and a proxy never puts such an id into /// an upstream URL: an id is pasted into URL paths, so `/`, `..`, `?`, diff --git a/crates/core/src/repo/hosted.rs b/crates/core/src/repo/hosted.rs index 97760d75..ba265961 100644 --- a/crates/core/src/repo/hosted.rs +++ b/crates/core/src/repo/hosted.rs @@ -876,6 +876,16 @@ impl HostedWrite { )) }; + // What the version's summary will be: an import adds what the source + // says of it (apt's component). + let mut manifest_summary = manifest_summary; + if let (Some(serde_json::Value::Object(summary)), Some(fields)) = ( + manifest_summary.as_mut(), + import.and_then(|options| options.summary_fields.as_ref()), + ) { + summary.extend(fields.clone()); + } + // An import of a package the source named otherwise stores nothing. if let Some((id, version)) = import.and_then(|options| options.expected.as_ref()) { let same_version = self @@ -908,6 +918,21 @@ impl HostedWrite { policy.authorize(context).await?; } + // A version past one of the format's caps is refused before anything + // reaches S3 (and an import's plan says so); the check in the + // transaction below stays the authority. + { + let mut conn = self.pool.acquire().await.map_err(RepoError::from)?; + self.check_summary_caps( + &mut conn, + repo_row.id, + &parsed, + manifest_summary.as_ref(), + false, + ) + .await?; + } + // A duplicate without redeploy is answered before anything reaches // S3: a CI job that re-pushes an unchanged version with // `--skip-duplicate` on every build would otherwise upload it each @@ -977,6 +1002,14 @@ impl HostedWrite { let mut retried = false; let (mut tx, component_id, replaced, outcome) = loop { let mut tx = forge_db::begin(&self.pool).await.map_err(RepoError::from)?; + self.check_summary_caps( + &mut tx, + repo_row.id, + &parsed, + manifest_summary.as_ref(), + true, + ) + .await?; // Read the duplicate-check *inside* the transaction so it serializes // with the subsequent insert/redeploy write against concurrent pushes @@ -1085,13 +1118,6 @@ impl HostedWrite { } }; - let mut manifest_summary = manifest_summary; - if let (Some(serde_json::Value::Object(summary)), Some(fields)) = ( - manifest_summary.as_mut(), - import.and_then(|options| options.summary_fields.as_ref()), - ) { - summary.extend(fields.clone()); - } if let (Some(id), Some(summary)) = (component_id, manifest_summary.as_ref()) { component_dao::set_manifest_summary(&mut *tx, id, summary) .await @@ -1151,6 +1177,69 @@ impl HostedWrite { done(outcome) } + /// Refuse a version whose summary would add a value to the repository + /// past one of the format's caps ([`Format::summary_caps`]: apt's + /// components), naming the cap. A value the repository's other versions + /// hold already (the one a redeploy replaces left out) passes. With + /// `lock`, in the publish transaction, a new value is counted under the + /// repository's value lock + /// ([`repository_dao::lock_repository_values`]), so that two publishes + /// of new values never both take the last place. + async fn check_summary_caps( + &self, + conn: &mut sqlx::PgConnection, + repository_id: i64, + parsed: &crate::domain::ParsedUpload, + summary: Option<&serde_json::Value>, + lock: bool, + ) -> Result<(), WriteError> { + for cap in self.format.summary_caps() { + let Some(value) = summary + .and_then(|s| s.get(cap.field)) + .and_then(|v| v.as_str()) + else { + continue; + }; + let mut values = component_dao::summary_values( + &mut *conn, + repository_id, + cap.field, + value, + &parsed.id, + &parsed.version, + ) + .await + .map_err(RepoError::from)?; + if !values.0 && lock { + repository_dao::lock_repository_values(&mut *conn, repository_id) + .await + .map_err(RepoError::from)?; + values = component_dao::summary_values( + &mut *conn, + repository_id, + cap.field, + value, + &parsed.id, + &parsed.version, + ) + .await + .map_err(RepoError::from)?; + } + let (present, count) = values; + if !present && usize::try_from(count).unwrap_or(usize::MAX) >= cap.max { + return Err(WriteError::InvalidPackage( + crate::domain::FormatError::InvalidPackage(format!( + "the repository holds {count} {what}, the most it may hold ({max}): \ + {value:?} would be one more", + what = cap.what, + max = cap.max, + )), + )); + } + } + Ok(()) + } + /// Insert a brand-new component + `.nupkg`/`.nuspec` assets, increfing both /// blobs, all within `tx`. #[allow(clippy::too_many_arguments)] diff --git a/crates/db/src/component_dao.rs b/crates/db/src/component_dao.rs index b8241b78..35d9170a 100644 --- a/crates/db/src/component_dao.rs +++ b/crates/db/src/component_dao.rs @@ -575,6 +575,45 @@ where Ok(()) } +/// The distinct values of the string field `field` of the manifest +/// summaries of `repository_id`'s versions, leaving out version `name` +/// `version` (one a redeploy replaces): whether `value` is among them, and +/// how many there are. A publish checks a format's caps on them +/// (`forge_plugin_api::SummaryCap`). +pub async fn summary_values<'e, E>( + executor: E, + repository_id: i64, + field: &str, + value: &str, + name: &str, + version: &str, +) -> sqlx::Result<(bool, i64)> +where + E: PgExecutor<'e>, +{ + // A runtime query: the field is a parameter, which the offline data of + // the macros checks no better. + let (present, count): (Option, i64) = sqlx::query_as( + r#" + SELECT bool_or(v = $3), COUNT(DISTINCT v) + FROM ( + SELECT manifest_summary->>$2 AS v + FROM component + WHERE repository_id = $1 AND NOT (name = $4 AND version = $5) + ) AS summary + WHERE v IS NOT NULL + "#, + ) + .bind(repository_id) + .bind(field) + .bind(value) + .bind(name) + .bind(version) + .fetch_one(executor) + .await?; + Ok((present.unwrap_or(false), count)) +} + /// Give an imported component the times its source recorded: `published_at` /// becomes its creation time, and `downloaded_at` its (and its package /// asset's, at `package_path`) last download unless Forge saw a later one. diff --git a/crates/db/src/repository_dao.rs b/crates/db/src/repository_dao.rs index 1a64b18b..aadddedf 100644 --- a/crates/db/src/repository_dao.rs +++ b/crates/db/src/repository_dao.rs @@ -650,6 +650,24 @@ where Ok(row.is_some()) } +/// Lock a repository row `FOR NO KEY UPDATE` inside the caller's +/// transaction: publishes that add a new value of a capped summary field +/// (`component_dao::summary_values`) take it before they count the values, +/// so that two of them never both pass the cap. It does not conflict with +/// the `FOR KEY SHARE` lock of every `component` insert, so other publishes +/// do not wait for it. Returns `false` when the row does not exist. +pub async fn lock_repository_values<'e, E>(executor: E, id: i64) -> sqlx::Result +where + E: sqlx::PgExecutor<'e>, +{ + let row: Option = + sqlx::query_scalar("SELECT id FROM repository WHERE id = $1 FOR NO KEY UPDATE") + .bind(id) + .fetch_optional(executor) + .await?; + Ok(row.is_some()) +} + /// Delete a repository by id. Returns whether a row was deleted. /// /// Every dependent row cascades via `ON DELETE CASCADE` — `group_member` rows diff --git a/crates/formats/apt/src/control.rs b/crates/formats/apt/src/control.rs index 64ba52b1..0c630a01 100644 --- a/crates/formats/apt/src/control.rs +++ b/crates/formats/apt/src/control.rs @@ -212,6 +212,235 @@ pub fn valid_package_name(name: &str) -> bool { }) } +/// The architectures dpkg knows, sorted: what `dpkg-architecture -L` lists +/// (dpkg 1.22.6, from its `cputable`, `ostable` and `tupletable`). A hosted +/// repository takes packages of these and of `all` alone +/// ([`known_architecture`]): each architecture adds its indexes to every +/// component's, so a writer must not add them at will (#761). +pub const ARCHITECTURES: &[&str] = &[ + "aix-powerpc", + "aix-ppc64", + "alpha", + "amd64", + "arc", + "arm", + "arm64", + "armeb", + "armel", + "armhf", + "darwin-amd64", + "darwin-arm", + "darwin-arm64", + "darwin-i386", + "darwin-powerpc", + "darwin-ppc64", + "dragonflybsd-amd64", + "freebsd-amd64", + "freebsd-arm", + "freebsd-arm64", + "freebsd-i386", + "freebsd-powerpc", + "freebsd-ppc64", + "hppa", + "hurd-amd64", + "hurd-i386", + "i386", + "ia64", + "kfreebsd-amd64", + "kfreebsd-i386", + "kopensolaris-amd64", + "kopensolaris-i386", + "loong64", + "m68k", + "mint-m68k", + "mips", + "mips64", + "mips64el", + "mips64r6", + "mips64r6el", + "mipsel", + "mipsn32", + "mipsn32el", + "mipsn32r6", + "mipsn32r6el", + "mipsr6", + "mipsr6el", + "musl-linux-alpha", + "musl-linux-amd64", + "musl-linux-arc", + "musl-linux-arm", + "musl-linux-arm64", + "musl-linux-armeb", + "musl-linux-armhf", + "musl-linux-hppa", + "musl-linux-i386", + "musl-linux-ia64", + "musl-linux-loong64", + "musl-linux-m68k", + "musl-linux-mips", + "musl-linux-mips64", + "musl-linux-mips64el", + "musl-linux-mips64r6", + "musl-linux-mips64r6el", + "musl-linux-mipsel", + "musl-linux-mipsr6", + "musl-linux-mipsr6el", + "musl-linux-nios2", + "musl-linux-or1k", + "musl-linux-powerpc", + "musl-linux-powerpcel", + "musl-linux-ppc64", + "musl-linux-ppc64el", + "musl-linux-riscv64", + "musl-linux-s390", + "musl-linux-s390x", + "musl-linux-sh3", + "musl-linux-sh3eb", + "musl-linux-sh4", + "musl-linux-sh4eb", + "musl-linux-sparc", + "musl-linux-sparc64", + "netbsd-alpha", + "netbsd-amd64", + "netbsd-arc", + "netbsd-arm", + "netbsd-arm64", + "netbsd-armeb", + "netbsd-hppa", + "netbsd-i386", + "netbsd-ia64", + "netbsd-loong64", + "netbsd-m68k", + "netbsd-mips", + "netbsd-mips64", + "netbsd-mips64el", + "netbsd-mips64r6", + "netbsd-mips64r6el", + "netbsd-mipsel", + "netbsd-mipsr6", + "netbsd-mipsr6el", + "netbsd-nios2", + "netbsd-or1k", + "netbsd-powerpc", + "netbsd-powerpcel", + "netbsd-ppc64", + "netbsd-ppc64el", + "netbsd-riscv64", + "netbsd-s390", + "netbsd-s390x", + "netbsd-sh3", + "netbsd-sh3eb", + "netbsd-sh4", + "netbsd-sh4eb", + "netbsd-sparc", + "netbsd-sparc64", + "nios2", + "openbsd-alpha", + "openbsd-amd64", + "openbsd-arc", + "openbsd-arm", + "openbsd-arm64", + "openbsd-armeb", + "openbsd-hppa", + "openbsd-i386", + "openbsd-ia64", + "openbsd-loong64", + "openbsd-m68k", + "openbsd-mips", + "openbsd-mips64", + "openbsd-mips64el", + "openbsd-mips64r6", + "openbsd-mips64r6el", + "openbsd-mipsel", + "openbsd-mipsr6", + "openbsd-mipsr6el", + "openbsd-nios2", + "openbsd-or1k", + "openbsd-powerpc", + "openbsd-powerpcel", + "openbsd-ppc64", + "openbsd-ppc64el", + "openbsd-riscv64", + "openbsd-s390", + "openbsd-s390x", + "openbsd-sh3", + "openbsd-sh3eb", + "openbsd-sh4", + "openbsd-sh4eb", + "openbsd-sparc", + "openbsd-sparc64", + "or1k", + "powerpc", + "powerpcel", + "powerpcspe", + "ppc64", + "ppc64el", + "riscv64", + "s390", + "s390x", + "sh3", + "sh3eb", + "sh4", + "sh4eb", + "solaris-amd64", + "solaris-i386", + "solaris-sparc", + "solaris-sparc64", + "sparc", + "sparc64", + "uclibc-linux-alpha", + "uclibc-linux-amd64", + "uclibc-linux-arc", + "uclibc-linux-arm", + "uclibc-linux-arm64", + "uclibc-linux-armeb", + "uclibc-linux-armel", + "uclibc-linux-hppa", + "uclibc-linux-i386", + "uclibc-linux-ia64", + "uclibc-linux-loong64", + "uclibc-linux-m68k", + "uclibc-linux-mips", + "uclibc-linux-mips64", + "uclibc-linux-mips64el", + "uclibc-linux-mips64r6", + "uclibc-linux-mips64r6el", + "uclibc-linux-mipsel", + "uclibc-linux-mipsr6", + "uclibc-linux-mipsr6el", + "uclibc-linux-nios2", + "uclibc-linux-or1k", + "uclibc-linux-powerpc", + "uclibc-linux-powerpcel", + "uclibc-linux-ppc64", + "uclibc-linux-ppc64el", + "uclibc-linux-riscv64", + "uclibc-linux-s390", + "uclibc-linux-s390x", + "uclibc-linux-sh3", + "uclibc-linux-sh3eb", + "uclibc-linux-sh4", + "uclibc-linux-sh4eb", + "uclibc-linux-sparc", + "uclibc-linux-sparc64", + "x32", +]; + +/// The most components a hosted repository holds. Each adds the indexes of +/// every architecture: with all of [`ARCHITECTURES`] and `all`, 8 +/// components make 8 × 207 `Packages` lists of 6 files each (the list, its +/// two compressions, and a by-hash copy of each), 9,936 files, below the +/// host's 10,000 an index may hold (#761). Debian and Ubuntu publish 4. +pub const MAX_COMPONENTS: usize = 8; + +/// Whether `arch` is an architecture a package may be built for: `all`, or +/// one of dpkg's [`ARCHITECTURES`]. A package's control file and an +/// import are checked against it; [`valid_architecture`] is the name's +/// syntax, which paths and ids are read with. +pub fn known_architecture(arch: &str) -> bool { + arch == "all" || ARCHITECTURES.binary_search(&arch).is_ok() +} + /// Whether `arch` is a binary package's architecture: `all`, or a Debian /// architecture name (lowercase letters, digits and `-`). `any` and /// `source` name none. @@ -432,6 +661,53 @@ mod tests { } } + #[test] + fn architectures_are_dpkgs() { + assert!( + ARCHITECTURES.windows(2).all(|w| w[0] < w[1]), + "ARCHITECTURES is sorted, without duplicates" + ); + assert!(ARCHITECTURES.iter().all(|a| valid_architecture(a))); + for known in [ + "all", + "amd64", + "arm64", + "armhf", + "i386", + "ppc64el", + "riscv64", + "s390x", + "hurd-i386", + "musl-linux-amd64", + "x32", + "loong64", + ] { + assert!(known_architecture(known), "{known}"); + } + // Syntactically architecture names, but not dpkg's (#761). + for unknown in [ + "a1", + "foo", + "amd65", + "linux-amd64", + "any", + "source", + "", + "AMD64", + ] { + assert!(!known_architecture(unknown), "{unknown}"); + } + } + + #[test] + fn the_worst_case_index_fits_the_hosts_limit() { + // Every component of every architecture and `all`: 6 files each, + // plus Release, InRelease, Release.gpg, key.asc and key.gpg. The + // host refuses an index of more than 10,000 files. + let files = MAX_COMPONENTS * (ARCHITECTURES.len() + 1) * 6 + 5; + assert!(files <= 10_000, "{files}"); + } + #[test] fn the_section_names_the_component() { assert_eq!(component_of(Some("contrib/net")).unwrap(), "contrib"); diff --git a/crates/formats/apt/src/import.rs b/crates/formats/apt/src/import.rs index f6317d4a..68e00b66 100644 --- a/crates/formats/apt/src/import.rs +++ b/crates/formats/apt/src/import.rs @@ -19,7 +19,7 @@ use std::io::{BufRead, BufReader, Read}; use forge_plugin_api::files; use sha2::{Digest, Sha256}; -use crate::control::{valid_architecture, valid_component, valid_package_name, Paragraph}; +use crate::control::{known_architecture, valid_component, valid_package_name, Paragraph}; use crate::version; /// The most a `Packages` index may inflate to. @@ -315,8 +315,11 @@ impl Entry { if let Err(e) = version::Version::parse(&self.version) { return Some(format!("{:?} is not a Debian version: {e}", self.version)); } - if !valid_architecture(&self.architecture) { - return Some(format!("{:?} is not an architecture", self.architecture)); + if !known_architecture(&self.architecture) { + return Some(format!( + "{:?} is not an architecture dpkg knows", + self.architecture + )); } if self.filename.is_empty() { return Some("the index names no Filename for it".into()); diff --git a/crates/formats/apt/src/lib.rs b/crates/formats/apt/src/lib.rs index af9d7db1..a82d4182 100644 --- a/crates/formats/apt/src/lib.rs +++ b/crates/formats/apt/src/lib.rs @@ -34,8 +34,8 @@ use forge_plugin_api::{ Action, Answer, Assets, ChangeRequest, Descriptor, FetchResult, FilePackage, FormatError, FormatOption, FormatPlugin, IndexFile, IndexInput, Loaded, OptionInput, PackageRef, PackageSource, ParsedPackage, Prereleases, PublishRequest, Query, Remote, Repository, - RepositoryKind, Request, Response, Storage, Upload, UpstreamAnswer, UpstreamOp, UpstreamStep, - UpstreamToken, VersionClass, + RepositoryKind, Request, Response, Storage, SummaryCap, Upload, UpstreamAnswer, UpstreamOp, + UpstreamStep, UpstreamToken, VersionClass, }; use crate::layout::{PoolPath, Target, KEY_ASC, KEY_GPG}; @@ -134,6 +134,16 @@ impl FormatPlugin for Apt { manifest::summarize(manifest) } + // Every component adds the `Packages` lists of every architecture to + // the indexes (#761); the architectures are dpkg's. + fn summary_caps(&self) -> &'static [SummaryCap] { + &[SummaryCap { + field: "component", + max: control::MAX_COMPONENTS, + what: "components", + }] + } + fn check_option( &self, _kind: RepositoryKind, diff --git a/crates/formats/apt/src/manifest.rs b/crates/formats/apt/src/manifest.rs index 09357b0f..e45ab579 100644 --- a/crates/formats/apt/src/manifest.rs +++ b/crates/formats/apt/src/manifest.rs @@ -80,9 +80,10 @@ impl Control { version::Version::parse(&version) .map_err(|e| FormatError::InvalidVersion(e.to_string()))?; let architecture = field("Architecture")?; - if !control::valid_architecture(&architecture) { + if !control::known_architecture(&architecture) { return Err(FormatError::MalformedManifest(format!( - "{architecture:?} is not an architecture" + "{architecture:?} is not an architecture: dpkg's architectures \ + (`dpkg-architecture -L`) and `all` are" ))); } let component = control::component_of(paragraph.first_line("Section")) @@ -218,6 +219,19 @@ mod tests { assert_eq!(Summary::parse(&json).unwrap(), summary); } + #[test] + fn the_component_is_capped() { + use forge_plugin_api::FormatPlugin; + // The host reads the capped field out of the stored summary. + let caps = crate::Apt.summary_caps(); + assert_eq!(caps.len(), 1); + assert_eq!(caps[0].max, control::MAX_COMPONENTS); + let summary = + summarize(b"Package: a1\nVersion: 1\nArchitecture: all\nSection: c9/x\n").unwrap(); + let json: serde_json::Value = serde_json::from_str(&summary).unwrap(); + assert_eq!(json[caps[0].field], "c9"); + } + #[test] fn a_control_file_needs_its_identity() { for (bad, why) in [ @@ -236,6 +250,15 @@ mod tests { b"Package: hello\nVersion: 1\nArchitecture: any\n", "not an architecture", ), + // Spelled as one, but not an architecture dpkg knows (#761). + ( + b"Package: hello\nVersion: 1\nArchitecture: a1\n", + "not an architecture", + ), + ( + b"Package: hello\nVersion: 1\nArchitecture: linux-amd64\n", + "not an architecture", + ), ( b"Package: hello\nVersion: 1\nArchitecture: amd64\nSection: ../x/y\n", "component", diff --git a/crates/plugin-api/src/lib.rs b/crates/plugin-api/src/lib.rs index 8859f281..b25f3550 100644 --- a/crates/plugin-api/src/lib.rs +++ b/crates/plugin-api/src/lib.rs @@ -137,6 +137,15 @@ pub trait FormatPlugin: Send + Sync + 'static { self.parse_package(&mut file) } + /// The fields of [`Self::manifest_summary`] of which a hosted repository + /// holds a bounded number of distinct values ([`SummaryCap`]). The host + /// refuses (400) a publish or an import of a version whose summary would + /// add a value past a cap, so whatever the repository's indexes make of + /// those values stays bounded. The default has none. + fn summary_caps(&self) -> &'static [SummaryCap] { + &[] + } + /// A JSON object summarizing `manifest`, stored with the version so that /// rendering never re-reads the manifest ([`PackageInfo::summary`]). /// `None` when there is nothing worth keeping. Every listing hands each diff --git a/crates/plugin-api/src/types.rs b/crates/plugin-api/src/types.rs index d7c97e6e..86d2773d 100644 --- a/crates/plugin-api/src/types.rs +++ b/crates/plugin-api/src/types.rs @@ -1558,3 +1558,19 @@ pub enum UpstreamStep { Fetch(Vec), Done(UpstreamAnswer), } + +/// A bound on the distinct values of one string field of the stored +/// summaries of a hosted repository's versions +/// ([`FormatPlugin::summary_caps`]): apt's components, each of which adds +/// its indexes to the repository's. +/// +/// [`FormatPlugin::summary_caps`]: crate::FormatPlugin::summary_caps +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SummaryCap { + /// The field of the summary's JSON object (`component`). + pub field: &'static str, + /// The most distinct values a repository may hold. + pub max: usize, + /// What the values are, plural, for the refusal (`components`). + pub what: &'static str, +} diff --git a/crates/plugin/src/format.rs b/crates/plugin/src/format.rs index e2ee010f..cc27d0cc 100644 --- a/crates/plugin/src/format.rs +++ b/crates/plugin/src/format.rs @@ -162,6 +162,10 @@ impl Format for UploadFormat { fn manifest_summary(&self, manifest: &[u8]) -> Option { self.format.manifest_summary(manifest) } + + fn summary_caps(&self) -> &'static [api::SummaryCap] { + self.format.summary_caps() + } } fn format_error(e: api::FormatError) -> FormatError { @@ -357,6 +361,10 @@ impl Format for PluginFormat { } } } + + fn summary_caps(&self) -> &'static [api::SummaryCap] { + self.plugin.summary_caps() + } } #[cfg(test)] diff --git a/crates/server/tests/apt.rs b/crates/server/tests/apt.rs index aabd0b3e..ea5b4d59 100644 --- a/crates/server/tests/apt.rs +++ b/crates/server/tests/apt.rs @@ -496,6 +496,77 @@ async fn hosted_repositories_publish_signed_indexes() { /// The by-hash copies of an index stay a day after a newer index replaces /// it, counted from the replacement, however old the index was (#762). +/// A writer cannot take a repository's indexes past what they may hold +/// (#761): an architecture dpkg does not know is refused, and so is a +/// package of a component past the cap, while the indexes keep working. +#[tokio::test] +async fn architectures_and_components_are_bounded() { + let srv = common::start().await; + let name = format!("apt{}", unique()); + create( + &srv, + json!({"name": name, "kind": "hosted", "format": "apt", "options": {"distribution": "bookworm"}}), + ) + .await; + let apt = Apt::new(&srv, &name).await; + let (status, text) = apt.post(&deb("hello", "1.0-1", "amd64", "utils")).await; + assert_eq!(status, StatusCode::CREATED, "{text}"); + + // An architecture of the right spelling that dpkg does not know. + let (status, text) = apt.post(&deb("odd", "1", "a1", "utils")).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{text}"); + assert!(text.contains("not an architecture"), "{text}"); + // dpkg's own, of any kind, are taken. + for arch in ["arm64", "musl-linux-riscv64", "all"] { + let (status, text) = apt.post(&deb("multi", "1", arch, "utils")).await; + assert_eq!(status, StatusCode::CREATED, "{arch}: {text}"); + } + + // `main` and 7 more components fill the repository. + for c in 1..8 { + let (status, text) = apt + .post(&deb(&format!("pkg{c}"), "1", "amd64", &format!("c{c}/x"))) + .await; + assert_eq!(status, StatusCode::CREATED, "c{c}: {text}"); + } + // A ninth is refused, naming the cap. + let (status, text) = apt.post(&deb("pkg8", "1", "amd64", "c8/x")).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{text}"); + assert!(text.contains("8 components"), "{text}"); + // A component the repository holds still takes packages. + let (status, text) = apt.post(&deb("more", "1", "i386", "c3/x")).await; + assert_eq!(status, StatusCode::CREATED, "{text}"); + + // The indexes still work: signed, listing the 8 components and the + // packages. + let keys = keys(&srv, &name).await; + let (message, signed) = in_release(&apt.text("dists/bookworm/InRelease").await); + message.verify(&public(&keys[0])).unwrap(); + assert!( + signed.contains("Components: c1 c2 c3 c4 c5 c6 c7 main\n"), + "{signed}" + ); + assert!(!signed.contains(" c8/"), "{signed}"); + assert!(!signed.contains("binary-a1/"), "{signed}"); + let packages = apt.text("dists/bookworm/main/binary-amd64/Packages").await; + assert!(packages.contains("Package: hello\n"), "{packages}"); + let packages = apt.text("dists/bookworm/c3/binary-i386/Packages").await; + assert!(packages.contains("Package: more\n"), "{packages}"); + assert!(apt.text("key.asc").await.contains("BEGIN PGP PUBLIC KEY")); + + // Deleting the last package of a component frees its place. + let (status, _) = apt.get("pool/c7/p/pkg7/pkg7_1_amd64.deb").await; + assert_eq!(status, StatusCode::OK); + let resp = apt + .req(reqwest::Method::DELETE, "pool/c7/p/pkg7/pkg7_1_amd64.deb") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + let (status, text) = apt.post(&deb("pkg8", "1", "amd64", "c8/x")).await; + assert_eq!(status, StatusCode::CREATED, "{text}"); +} + #[tokio::test] async fn by_hash_copies_stay_a_day_after_the_index_is_replaced() { let srv = common::start().await; diff --git a/docs/apt.md b/docs/apt.md index 51c3f7f3..40dd0269 100644 --- a/docs/apt.md +++ b/docs/apt.md @@ -57,6 +57,13 @@ package of `Section: contrib/net` is in `contrib`, any other in `main`. Its **architectures** are those of its packages; an `all` package is listed under every one of them (and under `binary-all`). +Every component adds a `Packages` list of every architecture to the +indexes, so both are bounded: an `Architecture` is `all` or one dpkg +knows (`dpkg-architecture -L`: `amd64`, `arm64`, `musl-linux-amd64`, …), +and a repository holds at most **8 components**. A package of another +architecture, or of a ninth component, is refused with `400` (an import +reports it as failed); the repository's indexes stay as they are. + An [import](import.md#apt) fills a hosted repository from another apt repository (Nexus, a static repository, another Forge): every `.deb` its `Release` lists, checked against the size and SHA-256 its index gives, in @@ -273,6 +280,10 @@ ordered by Debian version; the indexes follow on their next request. downloaded until it is deleted. - **Ed25519 keys only.** RSA and NIST-curve keys are refused: the OpenPGP library's code for them is not constant-time. +- **8 components, dpkg's architectures.** A hosted repository holds at + most 8 components, and packages of `all` and of the architectures dpkg + knows ([Hosted repositories](#hosted-repositories)), so that its indexes + stay within the 10,000 files an index may hold. - **Source packages** (`.dsc`, `Sources`) are not hosted; a proxy serves its upstream's. - **zstd levels 20 to 22.** dpkg-deb compresses the control member like the diff --git a/docs/changes/2.2.0/761.fixed.md b/docs/changes/2.2.0/761.fixed.md new file mode 100644 index 00000000..8e25bd8d --- /dev/null +++ b/docs/changes/2.2.0/761.fixed.md @@ -0,0 +1,7 @@ +A writer could break a hosted apt repository's indexes for every reader: +packages of many components and made-up architectures took the index past +the 10,000 files it may hold, and from then on `InRelease`, `Packages` and +`key.asc` answered 500. An `Architecture` is now `all` or one dpkg knows +(`dpkg-architecture -L`), and a repository holds at most 8 components: an +upload of another architecture, or of a ninth component, answers 400 +naming the limit, and an import reports such a package as failed. diff --git a/docs/format-plugins.md b/docs/format-plugins.md index af34fe34..9927d16c 100644 --- a/docs/format-plugins.md +++ b/docs/format-plugins.md @@ -203,6 +203,14 @@ manifest, and the package's tags. Every listing hands each version's summary to the plugin, so a summary should stay small. +A plugin bounds how many distinct values of a string field of its +summaries a hosted repository holds with `summary_caps` (a list of +[`SummaryCap`](../crates/plugin-api/src/types.rs): the field, the most +values, what they are called). A publish or an import whose summary would +add a value past a cap is refused with `400` naming it; a value the +repository holds already passes. apt caps its `component` at 8, since each +adds the indexes of every architecture. The default has no caps. + ## Files formats A format whose descriptor says `storage: Storage::Files` holds files under