diff --git a/crates/formats/apt/src/lib.rs b/crates/formats/apt/src/lib.rs index a82d4182..bc2aee9f 100644 --- a/crates/formats/apt/src/lib.rs +++ b/crates/formats/apt/src/lib.rs @@ -32,7 +32,7 @@ use std::cmp::Ordering; use forge_plugin_api::{ Action, Answer, Assets, ChangeRequest, Descriptor, FetchResult, FilePackage, FormatError, - FormatOption, FormatPlugin, IndexFile, IndexInput, Loaded, OptionInput, PackageRef, + FormatOption, FormatPlugin, IndexFile, IndexInput, ListedFile, Loaded, OptionInput, PackageRef, PackageSource, ParsedPackage, Prereleases, PublishRequest, Query, Remote, Repository, RepositoryKind, Request, Response, Storage, SummaryCap, Upload, UpstreamAnswer, UpstreamOp, UpstreamStep, UpstreamToken, VersionClass, @@ -184,6 +184,14 @@ impl FormatPlugin for Apt { } } + fn file_releases(&self, path: &str) -> Vec { + proxy::releases(path) + } + + fn file_listed(&self, release_path: &str, release: &[u8], path: &str) -> Option { + proxy::listed_in(release_path, release, path) + } + fn handle(&self, repo: &Repository, req: &Request) -> Action { let target = match layout::parse(&req.path) { Ok(target) => target, diff --git a/crates/formats/apt/src/proxy.rs b/crates/formats/apt/src/proxy.rs index 707f2975..d0a26cf1 100644 --- a/crates/formats/apt/src/proxy.rs +++ b/crates/formats/apt/src/proxy.rs @@ -7,6 +7,11 @@ //! A `by-hash//` copy (`MD5Sum`, `SHA1`, `SHA256` or //! `SHA512`) is checked against the digest its path names, and never //! changes once it matched. +//! The indexes but those copies are kept consistent with the +//! distribution's `InRelease` and `Release` ([`releases`], +//! [`listed_in`]): an upstream without `Acquire-By-Hash` has apt fetch +//! them at their own path, and checks each against the `InRelease` it +//! was served. //! * `pool/…`: packages, which never change at their path. Before a //! package is cached the host checks it against the size and SHA-256 the //! upstream's `Packages` index of its component and architecture gives it @@ -28,6 +33,9 @@ use forge_plugin_api::{ Fetch, FetchResult, Fetched, FileLocation, Freshness, Remote, UpstreamAnswer, UpstreamStep, }; +use forge_plugin_api::ListedFile; + +use crate::import::{cleartext, Release}; use crate::layout::{self, HashType, Target}; /// The most an index may inflate to while it is searched. @@ -100,6 +108,45 @@ pub fn file(remote: &Remote, path: &str, fetched: &[FetchResult]) -> UpstreamSte UpstreamStep::Done(UpstreamAnswer::FileAt(location)) } +/// The files a cached file of the distribution is kept consistent with: +/// for every file under `dists//` but its `by-hash` copies, +/// the distribution's `InRelease` (what apt reads first) and `Release`, +/// the file itself aside. +pub fn releases(path: &str) -> Vec { + let Ok(Target::Dists { + distribution, rest, .. + }) = layout::parse(path) + else { + return Vec::new(); + }; + if layout::is_by_hash(&rest) { + return Vec::new(); + } + ["InRelease", "Release"] + .into_iter() + .filter(|name| rest != *name) + .map(|name| format!("dists/{distribution}/{name}")) + .collect() +} + +/// The size and SHA-256 that `release` (the content of `release_path`, an +/// `InRelease` or a `Release` of [`releases`]`(path)`) lists for `path`. +pub fn listed_in(release_path: &str, release: &[u8], path: &str) -> Option { + let (dir, name) = release_path.rsplit_once('/')?; + let rest = path.strip_prefix(dir)?.strip_prefix('/')?; + let text = std::str::from_utf8(release).ok()?; + let text = match name { + "InRelease" => cleartext(text).ok()?, + "Release" => text.to_string(), + _ => return None, + }; + let listed = Release::parse(&text).ok()?.files.remove(rest)?; + Some(ListedFile { + size: listed.size, + sha256: listed.sha256, + }) +} + enum Lookup { Fetch(Fetch), Found { @@ -517,6 +564,74 @@ mod tests { assert_eq!((at.size, at.sha256), (Some(7), Some("d".repeat(64)))); } + #[test] + fn index_files_are_kept_consistent_with_the_release() { + assert_eq!( + releases("dists/bookworm/main/binary-amd64/Packages.gz"), + ["dists/bookworm/InRelease", "dists/bookworm/Release"] + ); + assert_eq!( + releases("dists/bookworm/Release.gpg"), + ["dists/bookworm/InRelease", "dists/bookworm/Release"] + ); + // Each of the two with the other. + assert_eq!( + releases("dists/bookworm/InRelease"), + ["dists/bookworm/Release"] + ); + assert_eq!( + releases("dists/bookworm/Release"), + ["dists/bookworm/InRelease"] + ); + // Content-addressed copies and packages never change. + let hash = "c".repeat(64); + assert!(releases(&format!( + "dists/bookworm/main/binary-amd64/by-hash/SHA256/{hash}" + )) + .is_empty()); + assert!(releases("pool/main/h/hello/hello_1.0_amd64.deb").is_empty()); + + let release = format!( + "Origin: Acme\nSuite: bookworm\nSHA256:\n {} 42 main/binary-amd64/Packages.gz\n", + "A".repeat(64) + ); + let path = "dists/bookworm/main/binary-amd64/Packages.gz"; + let listed = ListedFile { + size: 42, + sha256: "a".repeat(64), + }; + assert_eq!( + listed_in("dists/bookworm/Release", release.as_bytes(), path), + Some(listed.clone()) + ); + let in_release = format!( + "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n{release}\ + -----BEGIN PGP SIGNATURE-----\n\nxyz\n-----END PGP SIGNATURE-----\n" + ); + assert_eq!( + listed_in("dists/bookworm/InRelease", in_release.as_bytes(), path), + Some(listed) + ); + // Not listed, another distribution's, or no InRelease. + let unlisted = "dists/bookworm/main/binary-arm64/Packages.gz"; + assert_eq!( + listed_in("dists/bookworm/Release", release.as_bytes(), unlisted), + None + ); + assert_eq!( + listed_in( + "dists/trixie/Release", + release.as_bytes(), + "dists/bookworm/main/binary-amd64/Packages.gz" + ), + None + ); + assert_eq!( + listed_in("dists/bookworm/InRelease", release.as_bytes(), path), + None + ); + } + #[test] fn urls_append_encoded_segments() { assert_eq!( diff --git a/crates/plugin-api/src/lib.rs b/crates/plugin-api/src/lib.rs index b25f3550..c71db31b 100644 --- a/crates/plugin-api/src/lib.rs +++ b/crates/plugin-api/src/lib.rs @@ -342,6 +342,29 @@ pub trait FormatPlugin: Send + Sync + 'static { FilePackage::None } + /// Proxies that keep the upstream's files: the paths of the files that + /// are published together with the file at `path` and list it, the one + /// clients read first first (an apt distribution's `InRelease`, then + /// `Release`), never `path` itself. The host serves the cached file + /// consistent with them: it re-validates the file once one of them was + /// replaced after the file was last checked, and when the first of them + /// it holds lists the file with another size or digest + /// ([`Self::file_listed`]). The default: none, every file is re-validated + /// on its own clock. + fn file_releases(&self, path: &str) -> Vec { + let _ = path; + Vec::new() + } + + /// The size and SHA-256 that `release`, the content of the file at + /// `release_path` (one of [`Self::file_releases`] of `path`), lists for + /// the file at `path`; `None` when it does not list it, or cannot be + /// read. The default: none. + fn file_listed(&self, release_path: &str, release: &[u8], path: &str) -> Option { + let _ = (release_path, release, path); + None + } + /// Images formats: the response to a request the host refused, or to /// what failed while it served it outside a query ([`ImageError`]). /// The host adds its own headers (a Bearer challenge). The default is diff --git a/crates/plugin-api/src/types.rs b/crates/plugin-api/src/types.rs index 195ef39d..4b4e8f82 100644 --- a/crates/plugin-api/src/types.rs +++ b/crates/plugin-api/src/types.rs @@ -1446,6 +1446,17 @@ pub enum UpstreamAnswer { Failed(String), } +/// A file's size and SHA-256 as another file of the upstream lists it +/// ([`FormatPlugin::file_listed`]: an index file in apt's `Release`). +/// +/// [`FormatPlugin::file_listed`]: crate::FormatPlugin::file_listed +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListedFile { + pub size: u64, + /// Lowercase hex. + pub sha256: String, +} + /// Where an upstream keeps a file ([`UpstreamAnswer::FileAt`]), and what the /// plugin knows of it. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/server/src/files/proxy.rs b/crates/server/src/files/proxy.rs index ee60ead6..fff767d1 100644 --- a/crates/server/src/files/proxy.rs +++ b/crates/server/src/files/proxy.rs @@ -19,6 +19,16 @@ //! by the repository's `updated_at`, which *Invalidate cache* bumps). //! * Concurrent first fetches of one file share one upstream request per //! instance. +//! * A file the format publishes together with others that list it (apt's +//! indexes, which a distribution's `InRelease` lists with their sizes and +//! digests: `FormatPlugin::file_releases`) is served consistent with the +//! first of those the proxy holds, the one clients read: it is due too +//! once one of them was replaced after it was last checked, and when the +//! one served lists it as another file and was checked after it. When +//! the file was stored after that one was last checked, that one is +//! re-validated first. Each re-validation that leaves the two apart +//! (the upstream itself is) is not repeated until one of them changes or +//! is checked again. use std::collections::HashMap; use std::sync::{Arc, LazyLock, Mutex, PoisonError, Weak}; @@ -32,7 +42,9 @@ use forge_core::repo::proxy::{ use forge_db::model::Repository; use forge_db::oci_dao; use forge_db::raw_dao::{self, NewRawFile, RawFile, Redeploy}; -use forge_plugin::api::FileLocation; +use forge_plugin::api::{FileLocation, ListedFile}; +use forge_plugin::FormatPlugin; +use forge_storage::BlobRef; use forge_telemetry::proxy::{Kind as LookupKind, Lookup, Outcome}; use futures_util::TryStreamExt; use reqwest::header::{CONTENT_TYPE, ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED}; @@ -293,25 +305,200 @@ impl Files<'_> { /// [`Self::fresh`], marking what it does on `lookup`. async fn fresh_file(&self, repo: &Repository, file: RawFile, lookup: &mut Lookup) -> RawFile { let max_age = self.max_age(repo); - if file.immutable || max_age == 0 || !repo.online || !is_due(&file, max_age) { + if file.immutable || max_age == 0 || !repo.online { + return file; + } + let releases = Box::pin(self.releases_in_step(repo, &file)).await; + if !is_due(&file, max_age) && !releases.file_due(&file) { return file; } let gate = gate(repo.id, &file.path); - let _held = gate.lock().await; + let held = gate.lock().await; // Another request may have re-validated it meanwhile. let file = match raw_dao::find(&self.state.pool, repo.id, &file.path).await { - Ok(Some(current)) if !is_due(¤t, max_age) => return current, - Ok(Some(current)) => current, + Ok(Some(current)) => { + if !is_due(¤t, max_age) + && !Box::pin(self.releases(repo, ¤t)) + .await + .file_due(¤t) + { + return current; + } + current + } _ => file, }; - match self.revalidate(repo, &file, lookup).await { + let current = match self.revalidate(repo, &file, lookup).await { Ok(current) => current, Err(error) => { lookup.mark(Outcome::StaleServed); tracing::warn!(repo = %repo.name, path = %file.path, error = %error.message(), "file proxy: re-validation failed; serving the cached copy"); - file + return file; + } + }; + drop(held); + if current.blob_sha256 != file.blob_sha256 { + // New content may be newer than the release served with it. + Box::pin(self.releases_in_step(repo, ¤t)).await; + } + current + } + + /// The release files of the cached `file` ([`Releases`]), the one served + /// re-validated first when `file` was stored after it was last checked + /// and it lists `file` as another file. + async fn releases_in_step(&self, repo: &Repository, file: &RawFile) -> Releases { + let releases = Box::pin(self.releases(repo, file)).await; + let Some(release) = releases.release_due(file) else { + return releases; + }; + // In a task of its own, from a fresh stack: the request is deep + // already, and a re-validation as deep again. + let (state, principal, repo_owned, release, since) = ( + self.state.clone(), + self.principal.clone(), + repo.clone(), + release.clone(), + file.updated_at, + ); + let refreshed = tokio::spawn( + async move { + let files = Files { + state: &state, + principal, + }; + let mut lookup = Lookup::start(&repo_owned.name, LookupKind::File); + let span = lookup.span(); + files + .refresh_release(&repo_owned, release, since, &mut lookup) + .instrument(span) + .await; + let outcome = lookup.marked().unwrap_or(Outcome::Hit); + lookup.finish(outcome); } + .instrument(tracing::Span::current()), + ) + .await; + if let Err(error) = refreshed { + tracing::warn!(repo = %repo.name, %error, "file proxy: re-validation of a release failed"); } + Box::pin(self.releases(repo, file)).await + } + + /// Re-validate the cached `release`, unless it was checked at or after + /// `since` meanwhile. + async fn refresh_release( + &self, + repo: &Repository, + release: RawFile, + since: chrono::DateTime, + lookup: &mut Lookup, + ) { + let gate = gate(repo.id, &release.path); + let _held = gate.lock().await; + let release = match raw_dao::find(&self.state.pool, repo.id, &release.path).await { + Ok(Some(current)) => current, + Ok(None) => return, + Err(_) => release, + }; + if release.checked_at.is_some_and(|checked| checked >= since) { + return; + } + if let Err(error) = self.revalidate(repo, &release, lookup).await { + lookup.mark(Outcome::StaleServed); + tracing::warn!(repo = %repo.name, path = %release.path, error = %error.message(), "file proxy: re-validation of a release failed; serving the cached copy"); + } + } + + /// The release files of `file` the proxy holds, and what the first of + /// them lists for it ([`FormatPlugin::file_releases`]). + async fn releases(&self, repo: &Repository, file: &RawFile) -> Releases { + let none = Releases { + held: Vec::new(), + listed: None, + }; + let Some(plugin) = self.state.formats.plugin(&repo.format) else { + return none; + }; + let paths = plugin.file_releases(&file.path); + if paths.is_empty() { + return none; + } + let mut held = Vec::new(); + for path in paths { + match raw_dao::find(&self.state.pool, repo.id, &path).await { + Ok(Some(release)) => held.push(release), + Ok(None) => {} + Err(error) => { + tracing::warn!(repo = %repo.name, %path, %error, "file proxy: a release file could not be looked up"); + } + } + } + let listed = match held.first() { + Some(release) => Box::pin(self.listed(repo, &plugin, release, &file.path)).await, + None => None, + }; + Releases { held, listed } + } + + /// What `release` lists for the file at `path`, read once per content. + async fn listed( + &self, + repo: &Repository, + plugin: &Arc, + release: &RawFile, + path: &str, + ) -> Option { + let size = u64::try_from(release.size).ok()?; + if size > self.state.upstream.config().max_metadata_bytes { + return None; + } + let key = format!( + "{}:{}:{}:{path}", + repo.format, release.blob_sha256, release.path + ); + let found: Result, ()> = self + .state + .cache + .get_or_try_insert_local( + "files:release-listed", + &key, + Duration::from_secs(3600), + || { + let (store, plugin) = (self.state.blob_store.clone(), plugin.clone()); + let (blob_sha256, s3_key) = (release.blob_sha256.clone(), release.s3_key.clone()); + let (release_path, path) = (release.path.clone(), path.to_string()); + let repo = repo.name.clone(); + // In a task of its own, from a fresh stack: the object + // store's reads are deep, and the request is already. + async move { + let read = tokio::spawn( + async move { + let blob = BlobRef { + sha256: &blob_sha256, + s3_key: &s3_key, + size, + }; + let bytes = store.read_all(blob).await.map_err(|error| { + tracing::warn!(%repo, path = %release_path, %error, "file proxy: a release file could not be read"); + })?; + tokio::task::spawn_blocking(move || { + plugin + .file_listed(&release_path, &bytes, &path) + .map(|listed| (listed.size, listed.sha256)) + }) + .await + .map_err(|_| ()) + } + .instrument(tracing::Span::current()), + ); + read.await.map_err(|_| ())? + } + }, + ) + .await; + let (size, sha256) = found.ok().flatten()?; + Some(ListedFile { size, sha256 }) } /// How long a proxy's cached copy is served before it is re-validated: @@ -409,7 +596,7 @@ impl Files<'_> { return Ok(None); } let gate = gate(repo.id, path); - let _held = gate.lock().await; + let held = gate.lock().await; if let Some(file) = raw_dao::find(&self.state.pool, repo.id, path).await? { return Ok(Some(file)); } @@ -434,9 +621,60 @@ impl Files<'_> { ))); } lookup.mark(Outcome::Fetched); - self.store_response(repo, path, &location, response) - .await - .map(Some) + let file = self.store_response(repo, path, &location, response).await?; + drop(held); + // The release served may be older than the file. + if self.max_age(repo) > 0 { + Box::pin(self.releases_in_step(repo, &file)).await; + } + Ok(Some(file)) + } +} + +/// The files a cached file is published together with +/// ([`FormatPlugin::file_releases`]), as the proxy holds them. +struct Releases { + /// The ones cached, the one clients read first first. + held: Vec, + /// What the first of them lists for the file, when it lists it. + listed: Option, +} + +impl Releases { + /// Whether the one served lists `file` as another file. + fn apart(&self, file: &RawFile) -> bool { + self.listed.as_ref().is_some_and(|listed| { + u64::try_from(file.size).ok() != Some(listed.size) + || !listed.sha256.eq_ignore_ascii_case(&file.blob_sha256) + }) + } + + /// Whether `file` is due to be consistent with them: one of them was + /// replaced after it was last checked, or the one served lists it as + /// another file and was checked after it. + fn file_due(&self, file: &RawFile) -> bool { + let Some(checked) = file.checked_at else { + return true; + }; + self.held.iter().any(|release| release.updated_at > checked) + || (self.apart(file) + && self + .held + .first() + .and_then(|release| release.checked_at) + .is_some_and(|release| release > checked)) + } + + /// The one served, when it is due to be consistent with `file`: it + /// lists `file` as another file, and was last checked before `file`'s + /// content was stored. + fn release_due(&self, file: &RawFile) -> Option<&RawFile> { + let served = self.held.first()?; + (self.apart(file) + && served + .checked_at + .is_none_or(|checked| checked < file.updated_at)) + .then_some(served) } } diff --git a/crates/server/tests/apt.rs b/crates/server/tests/apt.rs index ca346013..83202f0b 100644 --- a/crates/server/tests/apt.rs +++ b/crates/server/tests/apt.rs @@ -1382,3 +1382,158 @@ async fn invalidate_cache_and_deletes_evict_checked_packages() { assert_eq!(apt.get(HELLO).await, (StatusCode::OK, hello2)); assert_eq!(seen(), 3); } + +/// A generation of a mirror without `Acquire-By-Hash` (reprepro's and +/// aptly's default): its `InRelease`, `Release`, `Release.gpg` and +/// `Packages.gz`, the indexes listing `Packages.gz` as it is. +fn generation(version: &str) -> [(&'static str, Vec); 4] { + let packages = format!( + "Package: hello\nVersion: {version}\nFilename: pool/main/h/hello/hello_{version}_amd64.deb\n\ + Size: 1\nSHA256: {}\n", + "0".repeat(64) + ); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(packages.as_bytes()).unwrap(); + let packages_gz = gz.finish().unwrap(); + let release = format!( + "Origin: Vendor\nSuite: bookworm\nVersion: {version}\nComponents: main\nArchitectures: amd64\n\ + SHA256:\n {} {} main/binary-amd64/Packages.gz\n", + sha256(&packages_gz), + packages_gz.len() + ); + let in_release = format!( + "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n{release}\ + -----BEGIN PGP SIGNATURE-----\n\nsigned {version}\n-----END PGP SIGNATURE-----\n" + ); + [ + ("dists/bookworm/InRelease", in_release.into_bytes()), + ("dists/bookworm/Release", release.into_bytes()), + ( + "dists/bookworm/Release.gpg", + format!("detached {version}").into_bytes(), + ), + ("dists/bookworm/main/binary-amd64/Packages.gz", packages_gz), + ] +} + +/// An upstream without `by-hash` has apt fetch `Packages` at its own path +/// and check it against the `InRelease` it was served: the proxy serves the +/// two from one generation, though apt re-validates only `InRelease` while +/// its lists are current, and each file has its own clock (#798). +#[tokio::test] +async fn proxies_serve_indexes_consistent_with_the_release() { + let srv = common::start().await; + let state = Arc::new(Mirror::default()); + let url = mirror(state.clone()).await; + let publish = |version: &str| { + let mut files = state.files.lock().unwrap(); + for (path, body) in generation(version) { + files.insert(format!("vendor/{path}"), body); + } + }; + publish("1"); + let name = format!("aptvendor{}", unique()); + create( + &srv, + json!({"name": name, "kind": "proxy", "format": "apt", "remote_url": format!("{url}/vendor"), + "metadata_ttl_seconds": 3600, "options": {"distribution": "bookworm"}}), + ) + .await; + let apt = Apt::new(&srv, &name).await; + // Only the file at `path` is due; the TTL keeps every other one. + let expire = |path: &'static str| { + let pool = srv.pool.clone(); + let name = name.clone(); + async move { + sqlx::query( + "UPDATE raw_file SET checked_at = NULL WHERE path = $2 \ + AND repository_id = (SELECT id FROM repository WHERE name = $1)", + ) + .bind(&name) + .bind(path) + .execute(&pool) + .await + .unwrap(); + } + }; + let current = |version: &str, path: &str| { + generation(version) + .into_iter() + .find(|(p, _)| *p == path) + .unwrap() + .1 + }; + let packages = "dists/bookworm/main/binary-amd64/Packages.gz"; + // What `Packages.gz` the `InRelease` served lists. + let listed = |in_release: &[u8]| { + let text = String::from_utf8(in_release.to_vec()).unwrap(); + let line = text + .lines() + .find(|line| line.ends_with(" main/binary-amd64/Packages.gz")) + .unwrap() + .to_string(); + let mut words = line.split_whitespace(); + ( + words.next().unwrap().to_string(), + words.next().unwrap().parse::().unwrap(), + ) + }; + + for path in [ + "dists/bookworm/InRelease", + "dists/bookworm/Release", + "dists/bookworm/Release.gpg", + packages, + ] { + assert_eq!(apt.get(path).await, (StatusCode::OK, current("1", path))); + } + + // The upstream publishes; a client re-validates only `InRelease`. + publish("2"); + expire("dists/bookworm/InRelease").await; + let (status, in_release) = apt.get("dists/bookworm/InRelease").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(in_release, current("2", "dists/bookworm/InRelease")); + // The `Packages.gz` served then is the one it lists. + let (status, body) = apt.get(packages).await; + assert_eq!(status, StatusCode::OK); + assert_eq!((sha256(&body), body.len()), listed(&in_release)); + assert_eq!(body, current("2", packages)); + // `Release` and `Release.gpg` are due once `InRelease` was replaced. + assert_eq!( + apt.get("dists/bookworm/Release").await.1, + current("2", "dists/bookworm/Release") + ); + assert_eq!( + apt.get("dists/bookworm/Release.gpg").await.1, + current("2", "dists/bookworm/Release.gpg") + ); + // Consistent again. Each replacement made the others due once more (a + // conditional GET each); then nothing more is asked of the upstream. + let all = [ + "dists/bookworm/InRelease", + "dists/bookworm/Release", + "dists/bookworm/Release.gpg", + packages, + ]; + for path in all { + assert_eq!(apt.get(path).await.1, current("2", path)); + } + let before = state.requests.load(Ordering::SeqCst); + for path in all { + assert_eq!(apt.get(path).await.1, current("2", path)); + } + assert_eq!(state.requests.load(Ordering::SeqCst), before); + + // The other way round: `Packages.gz` is re-validated first, and the + // `InRelease` served is then the one that lists it. + publish("3"); + expire(packages).await; + let (status, body) = apt.get(packages).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, current("3", packages)); + let (status, in_release) = apt.get("dists/bookworm/InRelease").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(in_release, current("3", "dists/bookworm/InRelease")); + assert_eq!((sha256(&body), body.len()), listed(&in_release)); +} diff --git a/docs/apt.md b/docs/apt.md index c80868aa..2a673b2d 100644 --- a/docs/apt.md +++ b/docs/apt.md @@ -246,6 +246,13 @@ deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] https://forge.exa - `InRelease`, `Release` and the other indexes are re-validated with the upstream once the repository's metadata TTL (`metadata_ttl_seconds`, 10 minutes by default) has passed. +- The indexes are served consistent with the `InRelease` served with them + (else `Release`), for an upstream without `Acquire-By-Hash` (reprepro's + and aptly's default), whose indexes apt fetches at their own path: an + index that `InRelease` lists with another size or SHA-256 is + re-validated before it is served, and once `InRelease` or `Release` is + replaced, the distribution's other indexes (`Release.gpg` too) are + re-validated on their next download. - A pool file is checked against the size and SHA-256 the upstream's `Packages` index of its component and architecture gives it (the distribution's `Packages.gz`, else `.xz`, else the plain one), when the diff --git a/docs/changes/2.2.0/798.fixed.md b/docs/changes/2.2.0/798.fixed.md new file mode 100644 index 00000000..dd5d7a97 --- /dev/null +++ b/docs/changes/2.2.0/798.fixed.md @@ -0,0 +1,10 @@ +An apt proxy of an upstream without `Acquire-By-Hash` (reprepro's and +aptly's default, packagecloud, many vendor repositories) could serve an +`InRelease` and a `Packages` from two different publishes after the +upstream published, and `apt-get update` failed with "Hash Sum mismatch" +until the metadata TTL passed. Each index was re-validated on its own +clock. An index that the `InRelease` served lists with another size or +SHA-256 is now re-validated before it is served, and once `InRelease` or +`Release` is replaced, the distribution's other indexes (`Release.gpg` +too) are re-validated on their next download. Debian and Ubuntu archives, +which apt reads by hash, are served as before. diff --git a/docs/format-plugins.md b/docs/format-plugins.md index ed013c77..fc8169bc 100644 --- a/docs/format-plugins.md +++ b/docs/format-plugins.md @@ -413,6 +413,12 @@ descriptor. The host then keeps the indexes of its hosted repositories: and a package's file whose package the path does not name (`FilePackage::Unknown`) only a reader's whom every package is visible to. + `file_releases` names the files a proxy's cached file is published + together with and listed in (an apt distribution's `InRelease` and + `Release`), and `file_listed` reads the size and SHA-256 one of them + lists for it: the host re-validates the file once one of them was + replaced after it was last checked, or when the one it serves lists the + file as another one. ## Format options