From 4db207b192b95c1e842d62c7167c422e0fc85692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:53:10 -0300 Subject: [PATCH] feat(storage): diff beacon states against their snapshot, not their parent A beacon non-anchor state was an xdelta3 delta against its parent, so a cold read folded up to an epoch of deltas in sequence, each over a full encoded state. Diffing every state against the nearest snapshot on its branch makes a cold read one snapshot plus one delta, whatever the interval. The writer keeps the current snapshot's bytes in memory instead of the parent's, and remembers which snapshot the last written state belongs to, so the common case reads nothing extra. On a miss (a restart, or a fork off an older block) it follows StateDiffs frames to the first root with no diff row, which is the snapshot on a beacon directory. The reader is unchanged: every frame names its own base, so chains written parent-linked by an earlier build still fold, and no DB_VERSION bump is needed. Diffs grow through the epoch, since each carries every change since its snapshot. Pruning finalized states keeps only a few epochs of them on disk. Lean keeps parent-linked diffs: its StateDiff regenerates historical_block_hashes from the parent link. (cherry picked from commit a5da3a2705b6bfe2646c7a022403ad1e408224e2) --- crates/common/types/src/beacon/fork.rs | 16 +- crates/storage/src/api/tables.rs | 8 +- crates/storage/src/beacon_state_delta.rs | 10 +- crates/storage/src/state_writer.rs | 236 ++++++++++++++++------- crates/storage/src/store.rs | 224 +++++++++++++++++++-- docs/data_storage.md | 21 +- 6 files changed, 421 insertions(+), 94 deletions(-) diff --git a/crates/common/types/src/beacon/fork.rs b/crates/common/types/src/beacon/fork.rs index c3ccd42a2..1d090ab9f 100644 --- a/crates/common/types/src/beacon/fork.rs +++ b/crates/common/types/src/beacon/fork.rs @@ -22,9 +22,10 @@ use crate::beacon::preset; const LEAN_SNAPSHOT_INTERVAL: u64 = 1_024; /// How many slots apart full state snapshots are written for the Beacon -/// Chain, in [`ForkName::snapshot_interval`]: one epoch, so a -/// reconstruction fold is bounded at one epoch of blocks plus a single -/// decode. +/// Chain, in [`ForkName::snapshot_interval`]: one epoch. A beacon read applies +/// one delta against its snapshot whatever the interval, so what this bounds +/// is how large a delta grows (it carries every change since its snapshot) and +/// how often a full snapshot is written. const BEACON_SNAPSHOT_INTERVAL: u64 = preset::SLOTS_PER_EPOCH; /// A named fork of the Beacon Chain, ordered oldest to newest, followed by @@ -188,10 +189,11 @@ impl ForkName { /// here rather than on `Config`: putting it in a config a chain agrees on /// would imply the two had to agree on it. /// - /// Lean's interval does not survive contact with a ~350 MB beacon state, - /// since the reconstruction fold would apply that many deltas; beacon - /// takes an epoch, so the fold is bounded at one epoch of blocks plus a - /// single decode. + /// Lean's interval does not survive contact with a ~350 MB beacon state: + /// a beacon diff carries every change since its snapshot, so a long + /// interval lets deltas grow large, and a snapshot is a full copy of the + /// state. Beacon takes an epoch, which bounds both; a read applies a single + /// delta whatever the interval, and decodes once. pub const fn snapshot_interval(self) -> u64 { match self { ForkName::Lean => LEAN_SNAPSHOT_INTERVAL, diff --git a/crates/storage/src/api/tables.rs b/crates/storage/src/api/tables.rs index f093a42ac..c7f4e90db 100644 --- a/crates/storage/src/api/tables.rs +++ b/crates/storage/src/api/tables.rs @@ -31,9 +31,11 @@ pub enum Table { States, /// State diffs: slot || root -> StateDiff /// - /// Parent-linked diff written for every non-anchor state, keyed like - /// `States`. Pruned below the finalized snapshot interval together with - /// `States`. See `get_state` for reconstruction. + /// Diff written for every non-anchor state, keyed like `States`. Lean's is + /// parent-linked; a beacon diff is taken against the nearest snapshot on + /// its branch, so a beacon state is one snapshot plus one delta. Pruned + /// below the finalized snapshot interval together with `States`. See + /// `get_state` for reconstruction. StateDiffs, /// Metadata: string keys -> various scalar values Metadata, diff --git a/crates/storage/src/beacon_state_delta.rs b/crates/storage/src/beacon_state_delta.rs index 44233b425..d03d8a88d 100644 --- a/crates/storage/src/beacon_state_delta.rs +++ b/crates/storage/src/beacon_state_delta.rs @@ -20,8 +20,14 @@ //! //! `insert_state`/`get_state` (in `store.rs`) call into this module for the //! beacon arm: `insert_state` writes a snapshot at anchors and a -//! [`frame`]d [`encode`] otherwise, and `get_state` walks the resulting -//! chain and [`decode`]s it back, via [`unframe`]. +//! [`frame`]d [`encode`] otherwise, and `get_state` follows the frame's base +//! and [`decode`]s it back, via [`unframe`]. +//! +//! The base of a delta is the nearest snapshot on the state's branch, not its +//! parent. A read is then one snapshot plus one delta, whatever the state's +//! distance from the snapshot; the price is that a delta carries every change +//! since the snapshot, so it grows across the interval. The frame names its +//! own base, so the reader does not care which of the two a given frame uses. use ethlambda_types::primitives::H256; diff --git a/crates/storage/src/state_writer.rs b/crates/storage/src/state_writer.rs index 723802b69..0a329ebe6 100644 --- a/crates/storage/src/state_writer.rs +++ b/crates/storage/src/state_writer.rs @@ -17,10 +17,12 @@ //! needs to hold in mind is: //! //! - **One worker, never a pool.** A beacon state's delta is computed against -//! its parent's *encoded bytes*, so the backend walks in [`read_state`] are -//! only safe because states are inserted parent-before-child *and committed -//! in that same order*. A single thread draining one FIFO channel is what -//! gives the second half of that for free; a pool would not. +//! the *encoded bytes of the nearest snapshot on its branch*, and that +//! snapshot, like every ancestor, must already be committed when the delta is +//! planned; the backend walks in [`read_state`] are likewise only safe because +//! states are inserted parent-before-child *and committed in that same order*. +//! A single thread draining one FIFO channel is what gives the second half of +//! that for free; a pool would not. //! - **The channel blocks when full**, at [`STATE_WRITE_QUEUE_CAPACITY`] //! entries, which is exactly what the write already did before it moved off //! the importer's thread: this can only ever degrade to that, never past it. @@ -61,7 +63,7 @@ use libssz::{SszDecode, SszEncode}; use lru::LruCache; use tracing::error; -use crate::api::{StorageBackend, StorageReadViewExt, Table}; +use crate::api::{StorageBackend, StorageReadView, StorageReadViewExt, Table}; use crate::beacon_state_delta; use crate::error::Error; use crate::state_codec::{decode_lean_state_value, decode_state_value, encode_state_value}; @@ -79,8 +81,10 @@ pub(crate) struct StateWrite { /// beacon state that is not an anchor. pub diff: Option>, /// Beacon only: the target's [`encode_state_value`] bytes, for the - /// writer's parent memo. `None` on the lean arm, which diffs in the field - /// domain and never reads a parent's bytes. + /// writer's snapshot memo. `Some` exactly when the write is a snapshot: + /// only a snapshot is ever a diff base, so a diff's own bytes are never + /// kept. `None` on the lean arm, which diffs in the field domain and never + /// reads a base's bytes. pub encoded: Option>, } @@ -248,6 +252,14 @@ fn rebase_onto_resident(cache: &StateCache, state: &mut BeaconState) { /// [`encode_state_value`]) by walking `StateDiffs` back to the nearest /// `States` snapshot and folding deltas forward, byte domain only. /// +/// Every frame names its own base, and that is all this walk follows. A diff +/// written by [`plan_beacon_state_write`] names the snapshot of its interval, so +/// the walk is one hop and the fold applies exactly one delta. A diff written +/// parent-linked by an earlier build names its parent instead, and the same walk +/// still reaches the snapshot and folds the longer chain. That is why moving the +/// base from the parent to the snapshot needed no `DB_VERSION` bump, and why +/// this function must keep following `base_root` rather than assume one hop. +/// /// Mirrors [`reconstruct_state`]'s walk-then-replay shape for lean, but /// cannot share its body: a beacon `StateDiffs` record is a /// [`beacon_state_delta::frame`]d byte delta, not a [`StateDiff`], so the @@ -255,9 +267,8 @@ fn rebase_onto_resident(cache: &StateCache, state: &mut BeaconState) { /// [`beacon_state_delta::unframe`] instead of a `StateDiff`'s own field. /// /// Hands the encoded bytes to `consume` rather than returning a decoded -/// [`BeaconState`], so that a caller that only needs the bytes (the writer -/// thread's `StateWriter::encoded_parent_bytes`, for its diff base) is never -/// made to pay for a decode it will not use. +/// [`BeaconState`], so that a caller that only needs the bytes is never made to +/// pay for a decode it will not use. /// [`Store::get_state`](crate::store::Store::get_state)'s beacon arm is the /// one caller that decodes, and it does so exactly once, after every delta /// has already been folded. @@ -315,6 +326,42 @@ pub(crate) fn reconstruct_beacon_state_bytes( } } +/// The root of the snapshot that `root`'s state belongs to, found by following +/// `StateDiffs` frames' `base_root` until a root has no diff row. +/// +/// On a beacon directory a state is written as a snapshot xor a diff (see +/// [`plan_beacon_state_write`]), so a root with no `StateDiffs` row is a +/// snapshot. `States` is deliberately not probed to decide that: on RocksDB a +/// hit there materializes the whole blob value, which is over a hundred +/// megabytes, just to answer yes. +/// +/// A diff written by this version names the snapshot directly, so the walk is +/// one hop. Following `base_root` all the way, rather than taking one hop and +/// trusting it, is what keeps a directory whose diffs were written +/// parent-linked by an older build working: that chain ends at the snapshot +/// too, only after more hops. +/// +/// `None` when a root on the way has no block row, which is how a root the +/// store never heard of looks. Takes the caller's view because opening a second +/// one while holding the first can deadlock on a backend whose read lock is not +/// reentrant. +pub(crate) fn nearest_snapshot_root(view: &dyn StorageReadView, root: &H256) -> Option { + let mut cursor = *root; + loop { + let slot = block_slot_in(view, Chain::Beacon, &cursor)?; + let key = encode_slot_root_key(slot, &cursor); + let base = view + .read_with(Table::StateDiffs, &key, |frame| { + beacon_state_delta::unframe(frame).0 + }) + .expect("read"); + match base { + Some(base_root) => cursor = base_root, + None => return Some(cursor), + } + } +} + /// Applies `records` (snapshot child first) to a borrowed `snapshot` and /// hands the result to `consume`: the snapshot itself when there is nothing /// to apply, otherwise the owned output of the last delta. @@ -417,7 +464,7 @@ pub(crate) fn reconstruct_state( /// base to diff against. /// /// Split out from the planning functions because the beacon arm has to know -/// the answer *before* deciding whether to read the parent's encoded bytes at +/// the answer *before* deciding whether to fetch a snapshot's encoded bytes at /// all: at an anchor it never diffs, and fetching a base it will not use would /// turn an avoided read into a mandatory one on every anchor. pub(crate) fn is_anchor(slot: u64, parent_slot: Option, interval: u64) -> bool { @@ -450,10 +497,14 @@ pub(crate) fn plan_lean_state_write(state: State, parent: &State) -> StateWrite /// The bytes a beacon post-state writes. /// -/// `parent` is `Some((parent_root, parent_encoded_bytes))` for a state that -/// diffs against its parent, and `None` for an anchor. Carrying the anchor -/// decision in this argument rather than in a separate flag is what keeps a -/// caller from fetching a base it will not use; see [`is_anchor`]. +/// `base` is `Some((snapshot_root, snapshot_encoded_bytes))` for a state that +/// diffs, and `None` for an anchor. The base is the nearest snapshot on the +/// state's branch, not its parent: a diff then carries every change since that +/// snapshot, and reading any state back is one snapshot plus one delta however +/// far into the interval it sits. The frame names the snapshot as its +/// `base_root`. Carrying the anchor decision in this argument rather than in a +/// separate flag is what keeps a caller from fetching a base it will not use; +/// see [`is_anchor`]. /// /// `slot` is passed rather than read off `state` because the caller has /// already computed it for [`is_anchor`], and because @@ -462,16 +513,16 @@ pub(crate) fn plan_lean_state_write(state: State, parent: &State) -> StateWrite pub(crate) fn plan_beacon_state_write( state: &BeaconState, slot: u64, - parent: Option<(H256, &[u8])>, + base: Option<(H256, &[u8])>, ) -> StateWrite { let target = encode_state_value(state); - match parent { + match base { None => StateWrite { snapshot: Some(target.clone()), diff: None, encoded: Some(target), }, - Some((parent_root, base)) => { + Some((snapshot_root, base)) => { let delta = beacon_state_delta::encode(&target, base); // Deliberately not weakened to a plain `assert`: this repo's // release-fast profile keeps debug assertions on in tests while @@ -484,11 +535,11 @@ pub(crate) fn plan_beacon_state_write( "a beacon state delta must decode back to its target" ); let target_len = target.len() as u64; - let framed = beacon_state_delta::frame(parent_root, slot, target_len, &delta); + let framed = beacon_state_delta::frame(snapshot_root, slot, target_len, &delta); StateWrite { snapshot: None, diff: Some(framed), - encoded: Some(target), + encoded: None, } } } @@ -536,13 +587,22 @@ struct StateWriter { chain: Chain, cache: Arc, pending: Arc, - /// The most recently encoded beacon state, so the beacon write path does - /// not re-encode a parent's whole SSZ on every import. + /// Root and [`encode_state_value`] bytes of the snapshot the newest beacon + /// diffs are taken against, so a diff does not re-read a snapshot of over a + /// hundred megabytes on every import. + /// + /// Set whenever this thread writes a beacon snapshot, and when it has to + /// load one from the backend (after a restart, or a fork off an older + /// interval). Thread-local and a plain `Option`, not a shared `Mutex`: + /// requests are drained in order by this one thread. + snapshot_memo: Option<(H256, Vec)>, + /// The last beacon state this thread wrote and the snapshot root it + /// belongs to (itself when it was a snapshot). /// - /// Thread-local and a plain `Option`, not a shared `Mutex`: requests are - /// drained in order by this one thread, so the parent of the state being - /// written is whatever this thread wrote last. - encoded_memo: Option<(H256, Vec)>, + /// Requests are drained in order, so the parent of the state being written + /// is almost always this one, which answers "which snapshot does my parent + /// belong to" without a read. + last_base: Option<(H256, H256)>, } impl StateWriter { @@ -588,13 +648,17 @@ impl StateWriter { let parent_root = beacon_state.latest_block_header().parent_root; let interval = beacon_state.fork_name().snapshot_interval(); let parent_slot = beacon_block_slot(self.backend.as_ref(), &parent_root); - let base = (!is_anchor(slot, parent_slot, interval)) - .then(|| self.encoded_parent_bytes(parent_root)); - plan_beacon_state_write( - beacon_state, - slot, - base.as_deref().map(|base| (parent_root, base)), - ) + let snapshot_root = (!is_anchor(slot, parent_slot, interval)) + .then(|| self.load_snapshot_base(parent_root)); + let base = snapshot_root.map(|root| { + let (memo_root, bytes) = self + .snapshot_memo + .as_ref() + .expect("load_snapshot_base fills the memo"); + debug_assert_eq!(*memo_root, root); + (root, bytes.as_slice()) + }); + plan_beacon_state_write(beacon_state, slot, base) } }; @@ -623,36 +687,68 @@ impl StateWriter { } batch.commit().expect("commit"); - if let Some(encoded) = write.encoded { - self.encoded_memo = Some((root, encoded)); + if matches!(&*request.state, BeaconState::Lean(_)) { + return; + } + match write.encoded { + // A snapshot is its own base, and the next diffs are taken against it. + Some(encoded) => { + self.snapshot_memo = Some((root, encoded)); + self.last_base = Some((root, root)); + } + None => { + let (snapshot_root, _) = self + .snapshot_memo + .as_ref() + .expect("a beacon diff is planned against the memoized snapshot"); + self.last_base = Some((root, *snapshot_root)); + } } } - /// The parent's [`encode_state_value`] bytes, for the beacon delta. + /// Makes `snapshot_memo` hold the snapshot that `parent_root`'s state + /// belongs to, and returns that snapshot's root. /// - /// Checks the memo first: requests are drained in order, so the parent is - /// almost always the state this thread wrote last, making this free. A - /// miss folds the diff chain, which produces these exact bytes as a - /// by-product, so nothing is decoded and then re-encoded to satisfy it. + /// The common case is free: `last_base` names the snapshot of the state + /// this thread wrote last, which is almost always the parent, and the memo + /// holds that snapshot's bytes. A miss (a restart, or a fork off an older + /// block) resolves the snapshot with [`nearest_snapshot_root`] and, if the + /// memo is for another snapshot, reads its `States` row and replaces the + /// memo. /// /// # Panics /// - /// If `parent_root` has no state. This is only reached once + /// If the snapshot has no row. This is only reached once /// [`beacon_block_slot`] has found a parent block, which is only ever true - /// once that parent's own state has already been written: this thread - /// committed it before pulling the request being served now. - fn encoded_parent_bytes(&self, parent_root: H256) -> Vec { - self.encoded_memo + /// once that parent's own state has already been written, and a state's + /// snapshot is committed before the state itself: this thread is the only + /// writer and drains requests in order. + fn load_snapshot_base(&mut self, parent_root: H256) -> H256 { + let snapshot_root = match self.last_base { + Some((last_root, snapshot_root)) if last_root == parent_root => snapshot_root, + _ => { + let view = self.backend.begin_read().expect("read view"); + nearest_snapshot_root(view.as_ref(), &parent_root) + .expect("parent state must exist to diff against") + } + }; + if self + .snapshot_memo .as_ref() - .filter(|(root, _)| *root == parent_root) - .map(|(_, bytes)| bytes.clone()) - .unwrap_or_else(|| { - reconstruct_beacon_state_bytes(self.backend.as_ref(), &parent_root, |bytes| { - bytes.into_owned() - }) - .expect("read parent state") - .expect("parent state must exist to diff against") - }) + .is_some_and(|(root, _)| *root == snapshot_root) + { + return snapshot_root; + } + let view = self.backend.begin_read().expect("read view"); + let slot = block_slot_in(view.as_ref(), Chain::Beacon, &snapshot_root) + .expect("the snapshot's block must exist"); + let key = encode_slot_root_key(slot, &snapshot_root); + let bytes = view + .read_with(Table::States, &key, <[u8]>::to_vec) + .expect("read") + .expect("the parent's snapshot must be committed before the parent itself"); + self.snapshot_memo = Some((snapshot_root, bytes)); + snapshot_root } } @@ -690,7 +786,8 @@ impl StateWriterHandle { chain, cache, pending, - encoded_memo: None, + snapshot_memo: None, + last_base: None, }; let join = std::thread::Builder::new() .name("state-writer".into()) @@ -867,13 +964,13 @@ mod tests { } #[test] - fn a_beacon_plan_with_no_parent_writes_a_snapshot_carrying_the_memo() { + fn a_beacon_plan_with_no_base_writes_a_snapshot_carrying_the_memo() { let state = BeaconState::Lean(base_state()); let slot = state.expect_lean().slot; let plan = plan_beacon_state_write(&state, slot, None); assert!( plan.snapshot.is_some(), - "no parent means no base to diff against" + "no base means nothing to diff against" ); assert!(plan.diff.is_none()); assert_eq!( @@ -889,30 +986,37 @@ mod tests { /// produced it. That keeps this test free of a multi-megabyte fixture. #[test] fn a_beacon_delta_decodes_back_to_its_target() { - let parent = base_state(); - let child = child_state(&parent, parent.slot + 1); - let parent_root = parent.latest_block_header.hash_tree_root(); + let snapshot = base_state(); + let child = child_state(&snapshot, snapshot.slot + 1); + let snapshot_root = snapshot.latest_block_header.hash_tree_root(); - let base = encode_state_value(&BeaconState::Lean(parent)); + let base = encode_state_value(&BeaconState::Lean(snapshot)); let slot = child.slot; let state = BeaconState::Lean(child); - let plan = plan_beacon_state_write(&state, slot, Some((parent_root, &base))); + let target = encode_state_value(&state); + let plan = plan_beacon_state_write(&state, slot, Some((snapshot_root, &base))); assert!( plan.snapshot.is_none(), - "a parent means a diff, not a snapshot" + "a base means a diff, not a snapshot" + ); + assert!( + plan.encoded.is_none(), + "only a snapshot is a diff base, so a diff keeps no memo bytes" ); let framed = plan.diff.expect("a non-anchor records a diff"); let (read_base_root, framed_slot, target_len, delta) = beacon_state_delta::unframe(&framed); - assert_eq!(read_base_root, parent_root); + assert_eq!( + read_base_root, snapshot_root, + "the frame names the snapshot it was diffed against" + ); assert_eq!( framed_slot, slot, "the frame carries the target state's slot" ); assert_eq!( beacon_state_delta::decode(delta, &base, target_len as usize), - plan.encoded - .expect("the beacon arm always memoizes its target"), + target ); } diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index ebf3f4e17..16d949614 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -3320,7 +3320,7 @@ impl Store { /// counterpart dispatches reads on `self.chain`). /// /// The byte-producing half of each arm, and the backend reads, the - /// commit and the parent-bytes memo, all live on the background writer in + /// commit and the snapshot memo, all live on the background writer in /// [`crate::state_writer`]; this method only builds the request and hands /// it off. /// @@ -3340,15 +3340,17 @@ impl Store { /// falls back to a snapshot read or a diff-chain reconstruction. /// /// Beacon: a byte-domain [`beacon_state_delta`](crate::beacon_state_delta) - /// diff, snapshotting at anchors, the same shape as lean's but working in + /// diff, snapshotting at anchors, similar to lean's but working in /// the SSZ byte domain rather than the field domain: a beacon validator /// registry breaks lean's [`StateDiff`](crate::state_diff::StateDiff)'s "`validators` never changes" /// assumption every epoch, so lean's diff cannot be reused as-is. The - /// parent is identified the same way lean's is, off the post-state's own - /// `latest_block_header.parent_root`, but its *encoded* bytes are what the - /// delta is computed against; the writer thread's parent-bytes lookup - /// (`StateWriter::encoded_parent_bytes`) explains how those are obtained - /// without an extra encode round trip. The store's first-ever beacon + /// delta is computed against the *encoded* bytes of the nearest snapshot + /// on the state's branch, not against its parent: the snapshot is found + /// from the post-state's own `latest_block_header.parent_root`, and every + /// diff in an interval names the same snapshot, so a cold read is one + /// snapshot plus one delta. The writer thread's `StateWriter::load_snapshot_base` + /// explains how that snapshot is found and kept without a read per import. + /// The store's first-ever beacon /// state (a bootstrap or checkpoint-sync anchor) has no parent block on /// record, so it is always a snapshot regardless of the interval math: /// there is no base to diff against. @@ -7359,15 +7361,16 @@ mod tests { } #[test] - fn a_cold_beacon_state_folds_every_delta_back_from_its_snapshot() { + fn a_cold_beacon_state_reads_back_as_its_snapshot_plus_one_delta() { let backend: Arc = Arc::new(InMemoryBackend::new()); let mut store = beacon_test_store(backend.clone()); let interval = ForkName::Electra.snapshot_interval(); // Slot 0 and slot `interval` are snapshots (no parent, then a boundary - // crossing), and every slot between them is a delta on its parent. So - // the reads cover a borrowed snapshot with nothing to fold, and chains - // of 1 up to `interval - 1` deltas applied to the borrowed snapshot. + // crossing), and every slot between them is a delta on the interval's + // snapshot. So the reads cover a borrowed snapshot with nothing to + // fold, and a single delta applied to the borrowed snapshot however + // deep into the interval the state sits. let mut expected = Vec::new(); let mut parent = H256::ZERO; for slot in 0..=interval { @@ -7401,13 +7404,208 @@ mod tests { state } + /// The root `insert_beacon_chain` gives the block at `slot`. + fn beacon_chain_root(slot: u64) -> H256 { + H256::from([(slot + 1) as u8; 32]) + } + + /// Inserts a linear beacon chain over `slots`, each block's parent being + /// the previous slot's (the first one's being `parent`), and returns the + /// last root. + fn insert_beacon_chain( + store: &mut Store, + slots: std::ops::RangeInclusive, + mut parent: H256, + ) -> H256 { + for slot in slots { + let root = beacon_chain_root(slot); + store + .insert_signed_block(root, beacon_test_block(slot, parent)) + .expect("insert block"); + store + .insert_state(root, beacon_test_state_with_parent(slot, parent)) + .expect("insert state"); + parent = root; + } + parent + } + + /// Every `StateDiffs` row on the backend as `(slot, root, base_root)`. + fn beacon_diff_bases(backend: &Arc) -> Vec<(u64, H256, H256)> { + let view = backend.begin_read().expect("read view"); + view.prefix_iterator(Table::StateDiffs, &[]) + .expect("iterator") + .map(|row| { + let (key, value) = row.expect("row"); + let slot = u64::from_be_bytes(key[..8].try_into().unwrap()); + let root = H256::from_slice(&key[8..]); + (slot, root, crate::beacon_state_delta::unframe(&value).0) + }) + .collect() + } + + #[test] + fn every_beacon_diff_in_an_interval_names_the_intervals_snapshot() { + let backend: Arc = Arc::new(InMemoryBackend::new()); + let mut store = beacon_test_store(backend.clone()); + let interval = ForkName::Electra.snapshot_interval(); + + insert_beacon_chain(&mut store, 0..=interval, H256::ZERO); + // Dropping the store joins the writer, which settles the backend. + drop(store); + + let diffs = beacon_diff_bases(&backend); + assert_eq!( + diffs.len() as u64, + interval - 1, + "slots 1..interval are diffs" + ); + for (slot, _, base_root) in diffs { + assert_eq!( + base_root, + beacon_chain_root(0), + "the diff at slot {slot} must name the snapshot, not its parent" + ); + } + } + + #[test] + fn two_children_of_a_mid_interval_block_both_diff_against_the_snapshot() { + let backend: Arc = Arc::new(InMemoryBackend::new()); + let mut store = beacon_test_store(backend.clone()); + let fork_parent = insert_beacon_chain(&mut store, 0..=5, H256::ZERO); + + // Two siblings of one parent: the second skips a slot, so the two + // states differ. The writer's last-written state is the first sibling, + // not the second's parent, so this is the fork-off-an-older-block path. + let first = H256::from([100u8; 32]); + let second = H256::from([101u8; 32]); + let first_state = beacon_test_state_with_parent(6, fork_parent); + let second_state = beacon_test_state_with_parent(7, fork_parent); + let expected = [ + (first, first_state.to_ssz()), + (second, second_state.to_ssz()), + ]; + store + .insert_signed_block(first, beacon_test_block(6, fork_parent)) + .expect("insert block"); + store + .insert_state(first, first_state) + .expect("insert state"); + store + .insert_signed_block(second, beacon_test_block(7, fork_parent)) + .expect("insert block"); + store + .insert_state(second, second_state) + .expect("insert state"); + drop(store); + + let bases = beacon_diff_bases(&backend); + for sibling in [first, second] { + let (_, _, base_root) = bases + .iter() + .find(|(_, root, _)| *root == sibling) + .expect("the sibling has a diff"); + assert_eq!(*base_root, beacon_chain_root(0)); + } + + let cold = beacon_test_store(backend); + for (root, encoded) in expected { + let state = cold.get_state(&root).expect("get").expect("present"); + assert_eq!(state.to_ssz(), encoded); + } + } + + #[test] + fn a_restart_mid_interval_keeps_diffing_against_the_snapshot() { + let backend: Arc = Arc::new(InMemoryBackend::new()); + let interval = ForkName::Electra.snapshot_interval(); + let half = interval / 2; + + let mut store = beacon_test_store(backend.clone()); + let tip = insert_beacon_chain(&mut store, 0..=half, H256::ZERO); + drop(store); + + // A fresh store has an empty snapshot memo and no last-written state, + // so the next write has to find its snapshot by walking the frames. + let mut store = beacon_test_store(backend.clone()); + insert_beacon_chain(&mut store, half + 1..=interval, tip); + drop(store); + + let diffs = beacon_diff_bases(&backend); + assert_eq!(diffs.len() as u64, interval - 1); + for (slot, _, base_root) in diffs { + assert_eq!(base_root, beacon_chain_root(0), "diff at slot {slot}"); + } + + let cold = beacon_test_store(backend); + for slot in 0..=interval { + let state = cold + .get_state(&beacon_chain_root(slot)) + .expect("get") + .expect("present"); + assert_eq!(state.slot(), slot); + } + } + + #[test] + fn the_snapshot_walk_follows_old_parent_linked_rows_to_the_snapshot() { + let backend: Arc = Arc::new(InMemoryBackend::new()); + let mut store = beacon_test_store(backend.clone()); + let (s, a, b) = ( + beacon_chain_root(10), + beacon_chain_root(11), + beacon_chain_root(12), + ); + store + .insert_signed_block(s, beacon_test_block(10, H256::ZERO)) + .expect("insert block"); + store + .insert_signed_block(a, beacon_test_block(11, s)) + .expect("insert block"); + store + .insert_signed_block(b, beacon_test_block(12, a)) + .expect("insert block"); + + // Hand-written rows: S is a snapshot (no diff row), A diffs against S, + // and B diffs against A, the way a parent-linked older build wrote it. + let mut batch = backend.begin_write().expect("write batch"); + let snapshot = vec![(encode_slot_root_key(10, &s), b"snapshot".to_vec())]; + batch.put_batch(Table::States, snapshot).expect("put"); + let diffs = vec![ + ( + encode_slot_root_key(11, &a), + crate::beacon_state_delta::frame(s, 11, 0, b""), + ), + ( + encode_slot_root_key(12, &b), + crate::beacon_state_delta::frame(a, 12, 0, b""), + ), + ]; + batch.put_batch(Table::StateDiffs, diffs).expect("put"); + batch.commit().expect("commit"); + + let view = backend.begin_read().expect("read view"); + for root in [s, a, b] { + assert_eq!( + crate::state_writer::nearest_snapshot_root(view.as_ref(), &root), + Some(s) + ); + } + assert_eq!( + crate::state_writer::nearest_snapshot_root(view.as_ref(), &H256::from([99u8; 32])), + None, + "a root with no block has no snapshot" + ); + } + #[test] fn a_beacon_state_reconstructs_across_a_whole_snapshot_interval() { let mut store = beacon_test_store(Arc::new(InMemoryBackend::new())); let interval = ForkName::Electra.snapshot_interval(); - // A chain one slot longer than the interval, so the walk crosses a - // snapshot boundary and the fold has real work to do. + // A chain one slot longer than the interval, so the reads cross a + // snapshot boundary and the deltas have real work to do. let mut roots = Vec::new(); let mut parent = H256::ZERO; for slot in 0..=interval { diff --git a/docs/data_storage.md b/docs/data_storage.md index 781acbecd..a88050a8b 100644 --- a/docs/data_storage.md +++ b/docs/data_storage.md @@ -258,8 +258,10 @@ no table of its own: it rides inside every `States` snapshot alongside ### StateDiffs -`slot ‖ root → StateDiff`. A parent-linked diff written for **every** -non-anchor state, pruned together with `States`. See [State Storage](#state-storage-snapshots--diffs) for what a +`slot ‖ root → StateDiff`. A diff written for **every** non-anchor state, +pruned together with `States`. Lean's is parent-linked; a beacon diff is a +byte delta against the nearest snapshot on its branch, so a beacon state is one +snapshot plus one delta. See [State Storage](#state-storage-snapshots--diffs) for what a diff contains and how states are rebuilt. ### Metadata @@ -623,6 +625,18 @@ or change predictably. Instead, `insert_state` writes: to its parent (~68 minutes at 4-second slots). This bounds any reconstruction walk to at most `SNAPSHOT_ANCHOR_INTERVAL` diff applications. +This is the lean arm. A beacon state is also a diff or, at an anchor, a +snapshot, but the diff is a byte delta (`beacon_state_delta`) taken against the +**nearest snapshot on its branch**, not against its parent, and its frame names +that snapshot as `base_root`. Reading a beacon state is therefore one snapshot +plus exactly one delta, however far into the interval the state sits; the cost +is that a delta carries every change since its snapshot, so it grows across +the interval. The reader follows each frame's `base_root` and does not assume +one hop, so a directory whose diffs were written parent-linked by an earlier +build still reads correctly (no `DB_VERSION` bump was needed), and a diff after +a restart or on a fork finds its snapshot by walking `StateDiffs` frames until +a root with no diff row. + A `StateDiff` stores only what cannot be recovered elsewhere: the target slot, justified/finalized checkpoints, and the justification fields (`justified_slots`, `justifications_roots`, `justifications_validators`, @@ -650,7 +664,8 @@ Reads go through `get_state`, which tries three levels: after importing its block — is a cache hit. 2. A full snapshot in `States`. 3. Reconstruction: walk `base_root` pointers back through `StateDiffs` until - a snapshot is found, then replay the diffs forward. + a snapshot is found, then replay the diffs forward. On lean that is a chain + of parent-linked diffs; on beacon it is one hop, to the interval's snapshot. ```text STATE RECONSTRUCTION