diff --git a/crates/common/types/src/beacon/fork.rs b/crates/common/types/src/beacon/fork.rs index c3ccd42a2..1d090ab9f 100644 --- a/crates/common/types/src/beacon/fork.rs +++ b/crates/common/types/src/beacon/fork.rs @@ -22,9 +22,10 @@ use crate::beacon::preset; const LEAN_SNAPSHOT_INTERVAL: u64 = 1_024; /// How many slots apart full state snapshots are written for the Beacon -/// Chain, in [`ForkName::snapshot_interval`]: one epoch, so a -/// reconstruction fold is bounded at one epoch of blocks plus a single -/// decode. +/// Chain, in [`ForkName::snapshot_interval`]: one epoch. A beacon read applies +/// one delta against its snapshot whatever the interval, so what this bounds +/// is how large a delta grows (it carries every change since its snapshot) and +/// how often a full snapshot is written. const BEACON_SNAPSHOT_INTERVAL: u64 = preset::SLOTS_PER_EPOCH; /// A named fork of the Beacon Chain, ordered oldest to newest, followed by @@ -188,10 +189,11 @@ impl ForkName { /// here rather than on `Config`: putting it in a config a chain agrees on /// would imply the two had to agree on it. /// - /// Lean's interval does not survive contact with a ~350 MB beacon state, - /// since the reconstruction fold would apply that many deltas; beacon - /// takes an epoch, so the fold is bounded at one epoch of blocks plus a - /// single decode. + /// Lean's interval does not survive contact with a ~350 MB beacon state: + /// a beacon diff carries every change since its snapshot, so a long + /// interval lets deltas grow large, and a snapshot is a full copy of the + /// state. Beacon takes an epoch, which bounds both; a read applies a single + /// delta whatever the interval, and decodes once. pub const fn snapshot_interval(self) -> u64 { match self { ForkName::Lean => LEAN_SNAPSHOT_INTERVAL, diff --git a/crates/storage/src/api/tables.rs b/crates/storage/src/api/tables.rs index f093a42ac..c7f4e90db 100644 --- a/crates/storage/src/api/tables.rs +++ b/crates/storage/src/api/tables.rs @@ -31,9 +31,11 @@ pub enum Table { States, /// State diffs: slot || root -> StateDiff /// - /// Parent-linked diff written for every non-anchor state, keyed like - /// `States`. Pruned below the finalized snapshot interval together with - /// `States`. See `get_state` for reconstruction. + /// Diff written for every non-anchor state, keyed like `States`. Lean's is + /// parent-linked; a beacon diff is taken against the nearest snapshot on + /// its branch, so a beacon state is one snapshot plus one delta. Pruned + /// below the finalized snapshot interval together with `States`. See + /// `get_state` for reconstruction. StateDiffs, /// Metadata: string keys -> various scalar values Metadata, diff --git a/crates/storage/src/beacon_state_delta.rs b/crates/storage/src/beacon_state_delta.rs index 44233b425..d03d8a88d 100644 --- a/crates/storage/src/beacon_state_delta.rs +++ b/crates/storage/src/beacon_state_delta.rs @@ -20,8 +20,14 @@ //! //! `insert_state`/`get_state` (in `store.rs`) call into this module for the //! beacon arm: `insert_state` writes a snapshot at anchors and a -//! [`frame`]d [`encode`] otherwise, and `get_state` walks the resulting -//! chain and [`decode`]s it back, via [`unframe`]. +//! [`frame`]d [`encode`] otherwise, and `get_state` follows the frame's base +//! and [`decode`]s it back, via [`unframe`]. +//! +//! The base of a delta is the nearest snapshot on the state's branch, not its +//! parent. A read is then one snapshot plus one delta, whatever the state's +//! distance from the snapshot; the price is that a delta carries every change +//! since the snapshot, so it grows across the interval. The frame names its +//! own base, so the reader does not care which of the two a given frame uses. use ethlambda_types::primitives::H256; diff --git a/crates/storage/src/state_writer.rs b/crates/storage/src/state_writer.rs index 723802b69..0a329ebe6 100644 --- a/crates/storage/src/state_writer.rs +++ b/crates/storage/src/state_writer.rs @@ -17,10 +17,12 @@ //! needs to hold in mind is: //! //! - **One worker, never a pool.** A beacon state's delta is computed against -//! its parent's *encoded bytes*, so the backend walks in [`read_state`] are -//! only safe because states are inserted parent-before-child *and committed -//! in that same order*. A single thread draining one FIFO channel is what -//! gives the second half of that for free; a pool would not. +//! the *encoded bytes of the nearest snapshot on its branch*, and that +//! snapshot, like every ancestor, must already be committed when the delta is +//! planned; the backend walks in [`read_state`] are likewise only safe because +//! states are inserted parent-before-child *and committed in that same order*. +//! A single thread draining one FIFO channel is what gives the second half of +//! that for free; a pool would not. //! - **The channel blocks when full**, at [`STATE_WRITE_QUEUE_CAPACITY`] //! entries, which is exactly what the write already did before it moved off //! the importer's thread: this can only ever degrade to that, never past it. @@ -61,7 +63,7 @@ use libssz::{SszDecode, SszEncode}; use lru::LruCache; use tracing::error; -use crate::api::{StorageBackend, StorageReadViewExt, Table}; +use crate::api::{StorageBackend, StorageReadView, StorageReadViewExt, Table}; use crate::beacon_state_delta; use crate::error::Error; use crate::state_codec::{decode_lean_state_value, decode_state_value, encode_state_value}; @@ -79,8 +81,10 @@ pub(crate) struct StateWrite { /// beacon state that is not an anchor. pub diff: Option>, /// Beacon only: the target's [`encode_state_value`] bytes, for the - /// writer's parent memo. `None` on the lean arm, which diffs in the field - /// domain and never reads a parent's bytes. + /// writer's snapshot memo. `Some` exactly when the write is a snapshot: + /// only a snapshot is ever a diff base, so a diff's own bytes are never + /// kept. `None` on the lean arm, which diffs in the field domain and never + /// reads a base's bytes. pub encoded: Option>, } @@ -248,6 +252,14 @@ fn rebase_onto_resident(cache: &StateCache, state: &mut BeaconState) { /// [`encode_state_value`]) by walking `StateDiffs` back to the nearest /// `States` snapshot and folding deltas forward, byte domain only. /// +/// Every frame names its own base, and that is all this walk follows. A diff +/// written by [`plan_beacon_state_write`] names the snapshot of its interval, so +/// the walk is one hop and the fold applies exactly one delta. A diff written +/// parent-linked by an earlier build names its parent instead, and the same walk +/// still reaches the snapshot and folds the longer chain. That is why moving the +/// base from the parent to the snapshot needed no `DB_VERSION` bump, and why +/// this function must keep following `base_root` rather than assume one hop. +/// /// Mirrors [`reconstruct_state`]'s walk-then-replay shape for lean, but /// cannot share its body: a beacon `StateDiffs` record is a /// [`beacon_state_delta::frame`]d byte delta, not a [`StateDiff`], so the @@ -255,9 +267,8 @@ fn rebase_onto_resident(cache: &StateCache, state: &mut BeaconState) { /// [`beacon_state_delta::unframe`] instead of a `StateDiff`'s own field. /// /// Hands the encoded bytes to `consume` rather than returning a decoded -/// [`BeaconState`], so that a caller that only needs the bytes (the writer -/// thread's `StateWriter::encoded_parent_bytes`, for its diff base) is never -/// made to pay for a decode it will not use. +/// [`BeaconState`], so that a caller that only needs the bytes is never made to +/// pay for a decode it will not use. /// [`Store::get_state`](crate::store::Store::get_state)'s beacon arm is the /// one caller that decodes, and it does so exactly once, after every delta /// has already been folded. @@ -315,6 +326,42 @@ pub(crate) fn reconstruct_beacon_state_bytes( } } +/// The root of the snapshot that `root`'s state belongs to, found by following +/// `StateDiffs` frames' `base_root` until a root has no diff row. +/// +/// On a beacon directory a state is written as a snapshot xor a diff (see +/// [`plan_beacon_state_write`]), so a root with no `StateDiffs` row is a +/// snapshot. `States` is deliberately not probed to decide that: on RocksDB a +/// hit there materializes the whole blob value, which is over a hundred +/// megabytes, just to answer yes. +/// +/// A diff written by this version names the snapshot directly, so the walk is +/// one hop. Following `base_root` all the way, rather than taking one hop and +/// trusting it, is what keeps a directory whose diffs were written +/// parent-linked by an older build working: that chain ends at the snapshot +/// too, only after more hops. +/// +/// `None` when a root on the way has no block row, which is how a root the +/// store never heard of looks. Takes the caller's view because opening a second +/// one while holding the first can deadlock on a backend whose read lock is not +/// reentrant. +pub(crate) fn nearest_snapshot_root(view: &dyn StorageReadView, root: &H256) -> Option { + let mut cursor = *root; + loop { + let slot = block_slot_in(view, Chain::Beacon, &cursor)?; + let key = encode_slot_root_key(slot, &cursor); + let base = view + .read_with(Table::StateDiffs, &key, |frame| { + beacon_state_delta::unframe(frame).0 + }) + .expect("read"); + match base { + Some(base_root) => cursor = base_root, + None => return Some(cursor), + } + } +} + /// Applies `records` (snapshot child first) to a borrowed `snapshot` and /// hands the result to `consume`: the snapshot itself when there is nothing /// to apply, otherwise the owned output of the last delta. @@ -417,7 +464,7 @@ pub(crate) fn reconstruct_state( /// base to diff against. /// /// Split out from the planning functions because the beacon arm has to know -/// the answer *before* deciding whether to read the parent's encoded bytes at +/// the answer *before* deciding whether to fetch a snapshot's encoded bytes at /// all: at an anchor it never diffs, and fetching a base it will not use would /// turn an avoided read into a mandatory one on every anchor. pub(crate) fn is_anchor(slot: u64, parent_slot: Option, interval: u64) -> bool { @@ -450,10 +497,14 @@ pub(crate) fn plan_lean_state_write(state: State, parent: &State) -> StateWrite /// The bytes a beacon post-state writes. /// -/// `parent` is `Some((parent_root, parent_encoded_bytes))` for a state that -/// diffs against its parent, and `None` for an anchor. Carrying the anchor -/// decision in this argument rather than in a separate flag is what keeps a -/// caller from fetching a base it will not use; see [`is_anchor`]. +/// `base` is `Some((snapshot_root, snapshot_encoded_bytes))` for a state that +/// diffs, and `None` for an anchor. The base is the nearest snapshot on the +/// state's branch, not its parent: a diff then carries every change since that +/// snapshot, and reading any state back is one snapshot plus one delta however +/// far into the interval it sits. The frame names the snapshot as its +/// `base_root`. Carrying the anchor decision in this argument rather than in a +/// separate flag is what keeps a caller from fetching a base it will not use; +/// see [`is_anchor`]. /// /// `slot` is passed rather than read off `state` because the caller has /// already computed it for [`is_anchor`], and because @@ -462,16 +513,16 @@ pub(crate) fn plan_lean_state_write(state: State, parent: &State) -> StateWrite pub(crate) fn plan_beacon_state_write( state: &BeaconState, slot: u64, - parent: Option<(H256, &[u8])>, + base: Option<(H256, &[u8])>, ) -> StateWrite { let target = encode_state_value(state); - match parent { + match base { None => StateWrite { snapshot: Some(target.clone()), diff: None, encoded: Some(target), }, - Some((parent_root, base)) => { + Some((snapshot_root, base)) => { let delta = beacon_state_delta::encode(&target, base); // Deliberately not weakened to a plain `assert`: this repo's // release-fast profile keeps debug assertions on in tests while @@ -484,11 +535,11 @@ pub(crate) fn plan_beacon_state_write( "a beacon state delta must decode back to its target" ); let target_len = target.len() as u64; - let framed = beacon_state_delta::frame(parent_root, slot, target_len, &delta); + let framed = beacon_state_delta::frame(snapshot_root, slot, target_len, &delta); StateWrite { snapshot: None, diff: Some(framed), - encoded: Some(target), + encoded: None, } } } @@ -536,13 +587,22 @@ struct StateWriter { chain: Chain, cache: Arc, pending: Arc, - /// The most recently encoded beacon state, so the beacon write path does - /// not re-encode a parent's whole SSZ on every import. + /// Root and [`encode_state_value`] bytes of the snapshot the newest beacon + /// diffs are taken against, so a diff does not re-read a snapshot of over a + /// hundred megabytes on every import. + /// + /// Set whenever this thread writes a beacon snapshot, and when it has to + /// load one from the backend (after a restart, or a fork off an older + /// interval). Thread-local and a plain `Option`, not a shared `Mutex`: + /// requests are drained in order by this one thread. + snapshot_memo: Option<(H256, Vec)>, + /// The last beacon state this thread wrote and the snapshot root it + /// belongs to (itself when it was a snapshot). /// - /// Thread-local and a plain `Option`, not a shared `Mutex`: requests are - /// drained in order by this one thread, so the parent of the state being - /// written is whatever this thread wrote last. - encoded_memo: Option<(H256, Vec)>, + /// Requests are drained in order, so the parent of the state being written + /// is almost always this one, which answers "which snapshot does my parent + /// belong to" without a read. + last_base: Option<(H256, H256)>, } impl StateWriter { @@ -588,13 +648,17 @@ impl StateWriter { let parent_root = beacon_state.latest_block_header().parent_root; let interval = beacon_state.fork_name().snapshot_interval(); let parent_slot = beacon_block_slot(self.backend.as_ref(), &parent_root); - let base = (!is_anchor(slot, parent_slot, interval)) - .then(|| self.encoded_parent_bytes(parent_root)); - plan_beacon_state_write( - beacon_state, - slot, - base.as_deref().map(|base| (parent_root, base)), - ) + let snapshot_root = (!is_anchor(slot, parent_slot, interval)) + .then(|| self.load_snapshot_base(parent_root)); + let base = snapshot_root.map(|root| { + let (memo_root, bytes) = self + .snapshot_memo + .as_ref() + .expect("load_snapshot_base fills the memo"); + debug_assert_eq!(*memo_root, root); + (root, bytes.as_slice()) + }); + plan_beacon_state_write(beacon_state, slot, base) } }; @@ -623,36 +687,68 @@ impl StateWriter { } batch.commit().expect("commit"); - if let Some(encoded) = write.encoded { - self.encoded_memo = Some((root, encoded)); + if matches!(&*request.state, BeaconState::Lean(_)) { + return; + } + match write.encoded { + // A snapshot is its own base, and the next diffs are taken against it. + Some(encoded) => { + self.snapshot_memo = Some((root, encoded)); + self.last_base = Some((root, root)); + } + None => { + let (snapshot_root, _) = self + .snapshot_memo + .as_ref() + .expect("a beacon diff is planned against the memoized snapshot"); + self.last_base = Some((root, *snapshot_root)); + } } } - /// The parent's [`encode_state_value`] bytes, for the beacon delta. + /// Makes `snapshot_memo` hold the snapshot that `parent_root`'s state + /// belongs to, and returns that snapshot's root. /// - /// Checks the memo first: requests are drained in order, so the parent is - /// almost always the state this thread wrote last, making this free. A - /// miss folds the diff chain, which produces these exact bytes as a - /// by-product, so nothing is decoded and then re-encoded to satisfy it. + /// The common case is free: `last_base` names the snapshot of the state + /// this thread wrote last, which is almost always the parent, and the memo + /// holds that snapshot's bytes. A miss (a restart, or a fork off an older + /// block) resolves the snapshot with [`nearest_snapshot_root`] and, if the + /// memo is for another snapshot, reads its `States` row and replaces the + /// memo. /// /// # Panics /// - /// If `parent_root` has no state. This is only reached once + /// If the snapshot has no row. This is only reached once /// [`beacon_block_slot`] has found a parent block, which is only ever true - /// once that parent's own state has already been written: this thread - /// committed it before pulling the request being served now. - fn encoded_parent_bytes(&self, parent_root: H256) -> Vec { - self.encoded_memo + /// once that parent's own state has already been written, and a state's + /// snapshot is committed before the state itself: this thread is the only + /// writer and drains requests in order. + fn load_snapshot_base(&mut self, parent_root: H256) -> H256 { + let snapshot_root = match self.last_base { + Some((last_root, snapshot_root)) if last_root == parent_root => snapshot_root, + _ => { + let view = self.backend.begin_read().expect("read view"); + nearest_snapshot_root(view.as_ref(), &parent_root) + .expect("parent state must exist to diff against") + } + }; + if self + .snapshot_memo .as_ref() - .filter(|(root, _)| *root == parent_root) - .map(|(_, bytes)| bytes.clone()) - .unwrap_or_else(|| { - reconstruct_beacon_state_bytes(self.backend.as_ref(), &parent_root, |bytes| { - bytes.into_owned() - }) - .expect("read parent state") - .expect("parent state must exist to diff against") - }) + .is_some_and(|(root, _)| *root == snapshot_root) + { + return snapshot_root; + } + let view = self.backend.begin_read().expect("read view"); + let slot = block_slot_in(view.as_ref(), Chain::Beacon, &snapshot_root) + .expect("the snapshot's block must exist"); + let key = encode_slot_root_key(slot, &snapshot_root); + let bytes = view + .read_with(Table::States, &key, <[u8]>::to_vec) + .expect("read") + .expect("the parent's snapshot must be committed before the parent itself"); + self.snapshot_memo = Some((snapshot_root, bytes)); + snapshot_root } } @@ -690,7 +786,8 @@ impl StateWriterHandle { chain, cache, pending, - encoded_memo: None, + snapshot_memo: None, + last_base: None, }; let join = std::thread::Builder::new() .name("state-writer".into()) @@ -867,13 +964,13 @@ mod tests { } #[test] - fn a_beacon_plan_with_no_parent_writes_a_snapshot_carrying_the_memo() { + fn a_beacon_plan_with_no_base_writes_a_snapshot_carrying_the_memo() { let state = BeaconState::Lean(base_state()); let slot = state.expect_lean().slot; let plan = plan_beacon_state_write(&state, slot, None); assert!( plan.snapshot.is_some(), - "no parent means no base to diff against" + "no base means nothing to diff against" ); assert!(plan.diff.is_none()); assert_eq!( @@ -889,30 +986,37 @@ mod tests { /// produced it. That keeps this test free of a multi-megabyte fixture. #[test] fn a_beacon_delta_decodes_back_to_its_target() { - let parent = base_state(); - let child = child_state(&parent, parent.slot + 1); - let parent_root = parent.latest_block_header.hash_tree_root(); + let snapshot = base_state(); + let child = child_state(&snapshot, snapshot.slot + 1); + let snapshot_root = snapshot.latest_block_header.hash_tree_root(); - let base = encode_state_value(&BeaconState::Lean(parent)); + let base = encode_state_value(&BeaconState::Lean(snapshot)); let slot = child.slot; let state = BeaconState::Lean(child); - let plan = plan_beacon_state_write(&state, slot, Some((parent_root, &base))); + let target = encode_state_value(&state); + let plan = plan_beacon_state_write(&state, slot, Some((snapshot_root, &base))); assert!( plan.snapshot.is_none(), - "a parent means a diff, not a snapshot" + "a base means a diff, not a snapshot" + ); + assert!( + plan.encoded.is_none(), + "only a snapshot is a diff base, so a diff keeps no memo bytes" ); let framed = plan.diff.expect("a non-anchor records a diff"); let (read_base_root, framed_slot, target_len, delta) = beacon_state_delta::unframe(&framed); - assert_eq!(read_base_root, parent_root); + assert_eq!( + read_base_root, snapshot_root, + "the frame names the snapshot it was diffed against" + ); assert_eq!( framed_slot, slot, "the frame carries the target state's slot" ); assert_eq!( beacon_state_delta::decode(delta, &base, target_len as usize), - plan.encoded - .expect("the beacon arm always memoizes its target"), + target ); } diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index ebf3f4e17..16d949614 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -3320,7 +3320,7 @@ impl Store { /// counterpart dispatches reads on `self.chain`). /// /// The byte-producing half of each arm, and the backend reads, the - /// commit and the parent-bytes memo, all live on the background writer in + /// commit and the snapshot memo, all live on the background writer in /// [`crate::state_writer`]; this method only builds the request and hands /// it off. /// @@ -3340,15 +3340,17 @@ impl Store { /// falls back to a snapshot read or a diff-chain reconstruction. /// /// Beacon: a byte-domain [`beacon_state_delta`](crate::beacon_state_delta) - /// diff, snapshotting at anchors, the same shape as lean's but working in + /// diff, snapshotting at anchors, similar to lean's but working in /// the SSZ byte domain rather than the field domain: a beacon validator /// registry breaks lean's [`StateDiff`](crate::state_diff::StateDiff)'s "`validators` never changes" /// assumption every epoch, so lean's diff cannot be reused as-is. The - /// parent is identified the same way lean's is, off the post-state's own - /// `latest_block_header.parent_root`, but its *encoded* bytes are what the - /// delta is computed against; the writer thread's parent-bytes lookup - /// (`StateWriter::encoded_parent_bytes`) explains how those are obtained - /// without an extra encode round trip. The store's first-ever beacon + /// delta is computed against the *encoded* bytes of the nearest snapshot + /// on the state's branch, not against its parent: the snapshot is found + /// from the post-state's own `latest_block_header.parent_root`, and every + /// diff in an interval names the same snapshot, so a cold read is one + /// snapshot plus one delta. The writer thread's `StateWriter::load_snapshot_base` + /// explains how that snapshot is found and kept without a read per import. + /// The store's first-ever beacon /// state (a bootstrap or checkpoint-sync anchor) has no parent block on /// record, so it is always a snapshot regardless of the interval math: /// there is no base to diff against. @@ -7359,15 +7361,16 @@ mod tests { } #[test] - fn a_cold_beacon_state_folds_every_delta_back_from_its_snapshot() { + fn a_cold_beacon_state_reads_back_as_its_snapshot_plus_one_delta() { let backend: Arc = Arc::new(InMemoryBackend::new()); let mut store = beacon_test_store(backend.clone()); let interval = ForkName::Electra.snapshot_interval(); // Slot 0 and slot `interval` are snapshots (no parent, then a boundary - // crossing), and every slot between them is a delta on its parent. So - // the reads cover a borrowed snapshot with nothing to fold, and chains - // of 1 up to `interval - 1` deltas applied to the borrowed snapshot. + // crossing), and every slot between them is a delta on the interval's + // snapshot. So the reads cover a borrowed snapshot with nothing to + // fold, and a single delta applied to the borrowed snapshot however + // deep into the interval the state sits. let mut expected = Vec::new(); let mut parent = H256::ZERO; for slot in 0..=interval { @@ -7401,13 +7404,208 @@ mod tests { state } + /// The root `insert_beacon_chain` gives the block at `slot`. + fn beacon_chain_root(slot: u64) -> H256 { + H256::from([(slot + 1) as u8; 32]) + } + + /// Inserts a linear beacon chain over `slots`, each block's parent being + /// the previous slot's (the first one's being `parent`), and returns the + /// last root. + fn insert_beacon_chain( + store: &mut Store, + slots: std::ops::RangeInclusive, + mut parent: H256, + ) -> H256 { + for slot in slots { + let root = beacon_chain_root(slot); + store + .insert_signed_block(root, beacon_test_block(slot, parent)) + .expect("insert block"); + store + .insert_state(root, beacon_test_state_with_parent(slot, parent)) + .expect("insert state"); + parent = root; + } + parent + } + + /// Every `StateDiffs` row on the backend as `(slot, root, base_root)`. + fn beacon_diff_bases(backend: &Arc) -> Vec<(u64, H256, H256)> { + let view = backend.begin_read().expect("read view"); + view.prefix_iterator(Table::StateDiffs, &[]) + .expect("iterator") + .map(|row| { + let (key, value) = row.expect("row"); + let slot = u64::from_be_bytes(key[..8].try_into().unwrap()); + let root = H256::from_slice(&key[8..]); + (slot, root, crate::beacon_state_delta::unframe(&value).0) + }) + .collect() + } + + #[test] + fn every_beacon_diff_in_an_interval_names_the_intervals_snapshot() { + let backend: Arc = Arc::new(InMemoryBackend::new()); + let mut store = beacon_test_store(backend.clone()); + let interval = ForkName::Electra.snapshot_interval(); + + insert_beacon_chain(&mut store, 0..=interval, H256::ZERO); + // Dropping the store joins the writer, which settles the backend. + drop(store); + + let diffs = beacon_diff_bases(&backend); + assert_eq!( + diffs.len() as u64, + interval - 1, + "slots 1..interval are diffs" + ); + for (slot, _, base_root) in diffs { + assert_eq!( + base_root, + beacon_chain_root(0), + "the diff at slot {slot} must name the snapshot, not its parent" + ); + } + } + + #[test] + fn two_children_of_a_mid_interval_block_both_diff_against_the_snapshot() { + let backend: Arc = Arc::new(InMemoryBackend::new()); + let mut store = beacon_test_store(backend.clone()); + let fork_parent = insert_beacon_chain(&mut store, 0..=5, H256::ZERO); + + // Two siblings of one parent: the second skips a slot, so the two + // states differ. The writer's last-written state is the first sibling, + // not the second's parent, so this is the fork-off-an-older-block path. + let first = H256::from([100u8; 32]); + let second = H256::from([101u8; 32]); + let first_state = beacon_test_state_with_parent(6, fork_parent); + let second_state = beacon_test_state_with_parent(7, fork_parent); + let expected = [ + (first, first_state.to_ssz()), + (second, second_state.to_ssz()), + ]; + store + .insert_signed_block(first, beacon_test_block(6, fork_parent)) + .expect("insert block"); + store + .insert_state(first, first_state) + .expect("insert state"); + store + .insert_signed_block(second, beacon_test_block(7, fork_parent)) + .expect("insert block"); + store + .insert_state(second, second_state) + .expect("insert state"); + drop(store); + + let bases = beacon_diff_bases(&backend); + for sibling in [first, second] { + let (_, _, base_root) = bases + .iter() + .find(|(_, root, _)| *root == sibling) + .expect("the sibling has a diff"); + assert_eq!(*base_root, beacon_chain_root(0)); + } + + let cold = beacon_test_store(backend); + for (root, encoded) in expected { + let state = cold.get_state(&root).expect("get").expect("present"); + assert_eq!(state.to_ssz(), encoded); + } + } + + #[test] + fn a_restart_mid_interval_keeps_diffing_against_the_snapshot() { + let backend: Arc = Arc::new(InMemoryBackend::new()); + let interval = ForkName::Electra.snapshot_interval(); + let half = interval / 2; + + let mut store = beacon_test_store(backend.clone()); + let tip = insert_beacon_chain(&mut store, 0..=half, H256::ZERO); + drop(store); + + // A fresh store has an empty snapshot memo and no last-written state, + // so the next write has to find its snapshot by walking the frames. + let mut store = beacon_test_store(backend.clone()); + insert_beacon_chain(&mut store, half + 1..=interval, tip); + drop(store); + + let diffs = beacon_diff_bases(&backend); + assert_eq!(diffs.len() as u64, interval - 1); + for (slot, _, base_root) in diffs { + assert_eq!(base_root, beacon_chain_root(0), "diff at slot {slot}"); + } + + let cold = beacon_test_store(backend); + for slot in 0..=interval { + let state = cold + .get_state(&beacon_chain_root(slot)) + .expect("get") + .expect("present"); + assert_eq!(state.slot(), slot); + } + } + + #[test] + fn the_snapshot_walk_follows_old_parent_linked_rows_to_the_snapshot() { + let backend: Arc = Arc::new(InMemoryBackend::new()); + let mut store = beacon_test_store(backend.clone()); + let (s, a, b) = ( + beacon_chain_root(10), + beacon_chain_root(11), + beacon_chain_root(12), + ); + store + .insert_signed_block(s, beacon_test_block(10, H256::ZERO)) + .expect("insert block"); + store + .insert_signed_block(a, beacon_test_block(11, s)) + .expect("insert block"); + store + .insert_signed_block(b, beacon_test_block(12, a)) + .expect("insert block"); + + // Hand-written rows: S is a snapshot (no diff row), A diffs against S, + // and B diffs against A, the way a parent-linked older build wrote it. + let mut batch = backend.begin_write().expect("write batch"); + let snapshot = vec![(encode_slot_root_key(10, &s), b"snapshot".to_vec())]; + batch.put_batch(Table::States, snapshot).expect("put"); + let diffs = vec![ + ( + encode_slot_root_key(11, &a), + crate::beacon_state_delta::frame(s, 11, 0, b""), + ), + ( + encode_slot_root_key(12, &b), + crate::beacon_state_delta::frame(a, 12, 0, b""), + ), + ]; + batch.put_batch(Table::StateDiffs, diffs).expect("put"); + batch.commit().expect("commit"); + + let view = backend.begin_read().expect("read view"); + for root in [s, a, b] { + assert_eq!( + crate::state_writer::nearest_snapshot_root(view.as_ref(), &root), + Some(s) + ); + } + assert_eq!( + crate::state_writer::nearest_snapshot_root(view.as_ref(), &H256::from([99u8; 32])), + None, + "a root with no block has no snapshot" + ); + } + #[test] fn a_beacon_state_reconstructs_across_a_whole_snapshot_interval() { let mut store = beacon_test_store(Arc::new(InMemoryBackend::new())); let interval = ForkName::Electra.snapshot_interval(); - // A chain one slot longer than the interval, so the walk crosses a - // snapshot boundary and the fold has real work to do. + // A chain one slot longer than the interval, so the reads cross a + // snapshot boundary and the deltas have real work to do. let mut roots = Vec::new(); let mut parent = H256::ZERO; for slot in 0..=interval { diff --git a/docs/data_storage.md b/docs/data_storage.md index 781acbecd..a88050a8b 100644 --- a/docs/data_storage.md +++ b/docs/data_storage.md @@ -258,8 +258,10 @@ no table of its own: it rides inside every `States` snapshot alongside ### StateDiffs -`slot ‖ root → StateDiff`. A parent-linked diff written for **every** -non-anchor state, pruned together with `States`. See [State Storage](#state-storage-snapshots--diffs) for what a +`slot ‖ root → StateDiff`. A diff written for **every** non-anchor state, +pruned together with `States`. Lean's is parent-linked; a beacon diff is a +byte delta against the nearest snapshot on its branch, so a beacon state is one +snapshot plus one delta. See [State Storage](#state-storage-snapshots--diffs) for what a diff contains and how states are rebuilt. ### Metadata @@ -623,6 +625,18 @@ or change predictably. Instead, `insert_state` writes: to its parent (~68 minutes at 4-second slots). This bounds any reconstruction walk to at most `SNAPSHOT_ANCHOR_INTERVAL` diff applications. +This is the lean arm. A beacon state is also a diff or, at an anchor, a +snapshot, but the diff is a byte delta (`beacon_state_delta`) taken against the +**nearest snapshot on its branch**, not against its parent, and its frame names +that snapshot as `base_root`. Reading a beacon state is therefore one snapshot +plus exactly one delta, however far into the interval the state sits; the cost +is that a delta carries every change since its snapshot, so it grows across +the interval. The reader follows each frame's `base_root` and does not assume +one hop, so a directory whose diffs were written parent-linked by an earlier +build still reads correctly (no `DB_VERSION` bump was needed), and a diff after +a restart or on a fork finds its snapshot by walking `StateDiffs` frames until +a root with no diff row. + A `StateDiff` stores only what cannot be recovered elsewhere: the target slot, justified/finalized checkpoints, and the justification fields (`justified_slots`, `justifications_roots`, `justifications_validators`, @@ -650,7 +664,8 @@ Reads go through `get_state`, which tries three levels: after importing its block — is a cache hit. 2. A full snapshot in `States`. 3. Reconstruction: walk `base_root` pointers back through `StateDiffs` until - a snapshot is found, then replay the diffs forward. + a snapshot is found, then replay the diffs forward. On lean that is a chain + of parent-linked diffs; on beacon it is one hop, to the interval's snapshot. ```text STATE RECONSTRUCTION